From f57e37f765e9f97bcb2e08514e2a6604b22975a9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:07:35 -0800 Subject: [PATCH 01/12] chore: remove the unused src/sample.py scratch file (ITL-1169) Co-Authored-By: Claude Opus 5.5 --- src/sample.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 src/sample.py diff --git a/src/sample.py b/src/sample.py deleted file mode 100644 index 3c10555b1..000000000 --- a/src/sample.py +++ /dev/null @@ -1,7 +0,0 @@ -from collections.abc import Mapping - - -def test() -> Mapping[str, type] | int: ... - - -x = test() From c7e5b2c5965fe5c1bdb4570173a0d133b4bd40d5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 14:09:50 -0800 Subject: [PATCH 02/12] chore: remove the superpowers/ directory and the rules that name it (ITL-1169) The hash audit example that docs/reference/hashing.md runs moves to scripts/. Plans go in the Linear issue or the pull request description, and the per-release checklist in the release pull request's description. Co-Authored-By: Claude Opus 5.5 --- .basedpyright/baseline.json | 806 ++--- .gitignore | 6 - .zed/rules | 10 +- AGENT_COORDINATION.md | 4 +- CLAUDE.md | 10 +- RELEASING.md | 3 +- docs/reference/hashing.md | 2 +- .../scripts => scripts}/hash_audit_example.py | 2 +- src/orcapod/pipeline/dag.py | 3 - .../plans/2026-05-12-data-to-data-rename.md | 614 ---- .../2026-05-20-pipeline-job-source-spec.md | 2742 --------------- .../plans/2026-05-20-polling-source.md | 1847 ---------- ...-05-21-eng-493-pipeline-pure-descriptor.md | 2995 ----------------- ...-22-eng-493-task6-pipelinejob-job-nodes.md | 487 --- ...3-eng-515-pipeline-pipelinejob-refactor.md | 1963 ----------- ...26-05-23-rename-config-to-orcapodconfig.md | 542 --- ...17-pipeline-job-execution-graph-cleanup.md | 1202 ------- .../2026-05-29-pipeline-dag-public-api.md | 1035 ------ ...6-06-02-dataclass-arrow-struct-encoding.md | 1158 ------- ...g-377-function-node-fetch-join-refactor.md | 864 ----- .../2026-06-03-pipeline-test-migration.md | 1133 ------- ...16-source-node-column-config-forwarding.md | 825 ----- ...26-06-08-eng-376-result-database-rename.md | 154 - ...-eng-389-empty-container-type-inference.md | 338 -- ...06-08-eng-571-tag-columns-normalization.md | 931 ----- .../2026-06-08-plt-964-persistent-config.md | 969 ------ .../2026-06-11-plt-1162-uuid-arrow-type.md | 956 ------ .../2026-06-11-type-specific-pod-access.md | 232 -- .../2026-06-13-datetime-arrow-timestamp.md | 438 --- ...-1652-extension-type-converter-protocol.md | 393 --- .../2026-06-14-extension-type-registry.md | 925 ----- .../2026-06-14-plt-1654-schema-walker.md | 660 ---- .../2026-06-14-plt-1655-database-hooks.md | 1300 ------- ...26-06-14-plt-1656-builtin-logical-types.md | 1511 --------- ...26-06-14-plt-1668-logical-type-redesign.md | 980 ------ ...lt-1670-orcapod-namespace-builtin-types.md | 721 ---- ...lt-1672-write-side-logical-type-factory.md | 1482 -------- ...t-1705-type-registration-spine-refactor.md | 2897 ---------------- ...ister-python-class-storage-type-cleanup.md | 831 ----- ...-plt-1731-pydantic-logical-type-factory.md | 1380 -------- ...01-wire-factories-into-default-registry.md | 571 ---- ...-06-21-plt-1737-schema-metadata-cleaner.md | 969 ------ ...ension-type-roundtrip-integration-tests.md | 827 ----- .../2026-06-23-pyspiral-upgrade-plt-1773.md | 316 -- ...lt-1660-hard-cut-extension-type-hashing.md | 2466 -------------- ...06-24-rename-semantic-hasher-to-handler.md | 422 --- ...3-merge-extension-type-system-into-main.md | 41 - .../plans/2026-06-28-pick-index-operators.md | 1558 --------- .../2026-06-29-itl-45-datetime-tag-values.md | 421 --- .../plans/2026-06-29-linear-release-system.md | 317 -- .../2026-06-29-typing-literal-support.md | 446 --- ...union-signature-hash-order-independence.md | 599 ---- superpowers/plans/2026-06-30-op-file-type.md | 1046 ------ .../plans/2026-06-30-release-workflow.md | 425 --- .../2026-06-30-union-typed-function-inputs.md | 448 --- ...01-itl-459-spikeinterface-baserecording.md | 764 ----- ...itl-460-numpy-ndarray-native-value-type.md | 684 ---- .../plans/2026-07-01-pandas-native-types.md | 976 ------ .../2026-07-02-itl-472-file-hash-caching.md | 1205 ------- .../2026-07-02-si-sorting-logical-type.md | 820 ----- .../plans/2026-07-02-spikeinterface-motion.md | 793 ----- .../plans/2026-07-03-itl-489-warm-cache.md | 899 ----- ...2026-07-03-itl-493-pathlike-conformance.md | 459 --- .../2026-07-03-itl-concurrent-warm-cache.md | 512 --- ...026-07-03-sorting-analyzer-logical-type.md | 714 ---- ...-itl-500-populate-hash-cache-ergonomics.md | 1496 -------- ...026-07-07-dict-source-auto-registration.md | 226 -- ...26-07-07-itl-142-pydantic-missing-tests.md | 205 -- ...7-itl-507-ephemeral-function-pod-output.md | 1788 ---------- ...71-spiraldb-arrow-metadata-preservation.md | 1593 --------- ...-08-itl-508-indexed-entry-id-versioning.md | 1301 ------- ...26-07-08-itl-511-directory-hash-caching.md | 836 ----- ...tl-473-orcapod-extension-spikeinterface.md | 1592 --------- ...2026-07-09-node-config-pod-config-split.md | 931 ----- .../2026-07-10-itl-516-unify-async-execute.md | 1357 -------- ...-07-10-itl-519-hash-cacher-write-guards.md | 923 ----- ...2026-07-10-itl-520-postgres-hash-cacher.md | 1016 ------ .../plans/2026-07-14-itl-523-post-run-hook.md | 1374 -------- .../2026-07-14-itl-525-side-effect-pod.md | 2311 ------------- ...-07-16-empty-data-ephemeral-propagation.md | 1589 --------- ...-07-16-itl-532-side-effect-function-pod.md | 2331 ------------- ...2026-07-16-itl-532-unified-function-pod.md | 1820 ---------- ...26-07-17-ctx-arg-and-ray-opts-hardening.md | 805 ----- ...-17-itl-535-schema-versioning-migration.md | 2666 --------------- ...l-533-remove-node-content-hash-preimage.md | 1199 ------- ...6-07-21-itl-545-release-branch-argument.md | 423 --- ...07-21-postrunpayload-invocation-context.md | 810 ----- .../2026-07-22-empty-table-nullability.md | 730 ---- ...05-itl-603-directory-exclude-by-pattern.md | 535 --- .../2026-08-07-npipe-204-groupby-operator.md | 1822 ---------- ...26-08-11-itl-173-list-set-logical-types.md | 1745 ---------- ...026-08-20-github-actions-node24-upgrade.md | 545 --- ...source-pipelinejob-async-delegation-fix.md | 327 -- ...6-08-22-itl-627-list-extension-metadata.md | 898 ----- ...polling-source-combine-content-hash-fix.md | 331 -- ...-pollingsource-concurrent-cursor-safety.md | 834 ----- ...g-952-polling-source-zero-row-batch-fix.md | 528 --- ...-itl-638-stable-type-annotation-hashing.md | 1245 ------- .../2026-09-03-itl-639-schema-hashing.md | 442 --- .../plans/2026-11-09-docs-gap-report.md | 300 -- .../2026-11-09-itl-134-remove-pygraphviz.md | 87 - .../2026-11-09-itl-810-v0.2-release-prep.md | 167 - ...026-11-09-itl-813-mergejoin-empty-input.md | 186 - ...6-11-09-itl-816-iter-data-chunk-layouts.md | 153 - ...-11-09-itl-817-system-tag-run-stability.md | 108 - ...itl-825-python-313-canonical-type-names.md | 132 - .../2026-11-09-itl-826-org-required-ci.md | 173 - .../2026-11-09-itl-827-packaging-cleanup.md | 162 - ...026-11-09-itl-832-mergejoin-arrow-types.md | 160 - ...26-11-09-itl-834-index-pick-arrow-types.md | 124 - .../plans/2026-11-09-plt-4816-docs-audit.md | 188 -- .../2026-11-09-pr-272-review-revision.md | 120 - .../2026-11-09-pr-275-coderabbit-review.md | 59 - .../2026-11-09-v0.2-release-checklist.md | 554 --- .../2026-11-09-workshop-facilitator-notes.md | 175 - .../plans/2026-11-09-workshop-tutorial.md | 220 -- .../plans/2026-11-10-itl-578-golden-merge.md | 29 - ...0-itl-578-polars-dataframe-logical-type.md | 302 -- .../2026-11-10-itl-578-polars-hash-labels.md | 86 - ...-11-10-itl-626-golden-generator-timeout.md | 77 - ...26-11-10-itl-626-golden-hash-regression.md | 166 - ...1-10-itl-810-finalize-v0.2-release-docs.md | 93 - ...0-itl-810-refresh-pr-269-public-release.md | 71 - ...tl-813-832-mergejoin-arrow-types-rework.md | 146 - ...26-11-10-itl-837-join-async-arrow-types.md | 150 - .../2026-11-10-itl-845-per-row-source-info.md | 190 -- ...6-11-10-itl-847-semijoin-nested-columns.md | 115 - ...itl-849-chained-aggregation-source-info.md | 206 -- ...6-11-10-itl-854-join-async-meta-columns.md | 58 - ...26-11-10-itl-864-delta-read-your-writes.md | 174 - .../plans/2026-11-10-itl-875-mit-license.md | 88 - .../2026-11-10-itl-878-store-write-failure.md | 306 -- ...-11-10-itl-881-user-class-identity-impl.md | 189 -- ...tl-884-coderabbit-flush-and-async-group.md | 165 - .../2026-11-10-itl-884-delta-read-failures.md | 210 -- .../plans/2026-11-10-itl-888-array-format.md | 275 -- .../plans/2026-11-10-itl-892-exit-hang.md | 295 -- .../2026-11-10-pr-281-public-call-tests.md | 65 - .../2026-11-10-pr-282-merge-main-itl-849.md | 21 - ...1-10-pr-293-coderabbit-id-cache-refresh.md | 59 - ...26-11-10-pr280-review-shared-arrow-type.md | 122 - .../2026-11-11-itl-578-merge-main-semijoin.md | 78 - .../2026-11-11-itl-821-newcomer-papercuts.md | 172 - ...2026-11-11-itl-828-polars-expr-identity.md | 87 - .../plans/2026-11-11-itl-864-stack-on-290.md | 60 - .../plans/2026-11-11-itl-871-torch-tensor.md | 311 -- ...-11-11-itl-875-public-release-readiness.md | 98 - ...2026-11-11-itl-881-field-order-identity.md | 99 - ...1-11-itl-884-no-message-only-parameters.md | 105 - .../plans/2026-11-11-itl-895-decisions.md | 165 - .../plans/2026-11-11-itl-895-float-to-int.md | 204 -- ...6-11-11-itl-896-async-operator-readback.md | 144 - ...2026-11-11-itl-897-pandas-hash-identity.md | 190 -- .../2026-11-11-itl-904-loaded-job-readback.md | 237 -- .../2026-11-11-itl-905-dict-order-hash.md | 256 -- .../2026-11-11-itl-910-adopt-starfix-05.md | 249 -- ...-itl-910-followup-ignore-old-label-rows.md | 125 - ...026-11-11-itl-919-delta-batched-commits.md | 252 -- ...26-11-11-itl-919-in-memory-store-linear.md | 123 - ...26-11-11-itl-926-spiral-source-reverify.md | 58 - ...-11-11-itl-935-logical-type-identifiers.md | 241 -- ...11-11-itl-936-spiral-nullability-record.md | 255 -- ...26-11-11-itl-938-post-session-decisions.md | 25 - ...-11-itl-939-derived-source-tags-as-data.md | 130 - ...11-itl-939-tag-and-data-same-name-spike.md | 101 - .../2026-11-11-itl-942-wrong-hash-fixes.md | 266 -- .../plans/2026-11-11-itl-943-drop-nulls.md | 171 - .../2026-11-11-itl-944-data-context-v1.md | 206 -- .../2026-11-11-itl-945-hash-label-rule.md | 263 -- ...itl-953-function-defaults-semantic-hash.md | 290 -- .../plans/2026-11-11-itl-954-prefix-scan.md | 118 - ...26-11-11-itl-955-groupby-tags-stay-tags.md | 233 -- .../2026-11-11-itl-957-default-source-ids.md | 380 --- .../2026-11-11-itl-961-changelog-fragments.md | 77 - .../2026-11-11-itl-967-hash-step-combined.md | 167 - ...tl-968-design-issues-one-file-per-entry.md | 163 - .../2026-11-12-itl-1001-flaky-async-run.md | 71 - .../2026-11-12-itl-1007-public-release.md | 13 - ...26-11-12-itl-1026-orcapod-version-stamp.md | 25 - .../2026-11-12-itl-1027-map-keep-default.md | 11 - ...2026-11-12-itl-1034-row-alignment-suite.md | 201 -- .../2026-11-12-itl-1039-resumed-iteration.md | 30 - ...-12-itl-1040-dropped-column-source-info.md | 28 - .../2026-11-12-itl-1042-head-after-async.md | 181 - ...026-11-12-itl-1043-loaded-operator-node.md | 167 - .../2026-11-12-itl-186-notebook-preview.md | 73 - ...6-11-12-itl-186-streaming-spiral-source.md | 154 - ...1-12-itl-578-refresh-onto-v03-hash-step.md | 136 - ...-11-12-itl-807-function-pod-auto-select.md | 127 - ...1-12-itl-871-native-storage-strict-hash.md | 196 -- .../plans/2026-11-12-itl-871-refresh.md | 125 - .../plans/2026-11-12-itl-895-refresh.md | 98 - ...026-11-12-itl-895-review-round-c7f53bc0.md | 37 - ...1-12-itl-897-refresh-onto-v03-hash-step.md | 208 -- .../plans/2026-11-12-itl-904-refresh.md | 91 - ...2026-11-12-itl-905-refresh-against-main.md | 87 - ...26-11-12-itl-906-async-system-tag-names.md | 150 - .../2026-11-12-itl-928-node-tag-types.md | 140 - ...26-11-12-itl-943-drop-nulls-all-columns.md | 115 - .../2026-11-12-itl-967-golden-entries.md | 1116 ------ .../2026-11-12-itl-972-user-class-kinds.md | 136 - .../2026-11-12-itl-975-hash-step-follow-up.md | 176 - .../2026-11-12-itl-981-library-value-types.md | 96 - .../2026-11-12-itl-984-delta-second-commit.md | 23 - .../2026-11-12-itl-988-cached-pod-job.md | 20 - .../plans/2026-11-12-itl-991-stream-head.md | 221 -- .../plans/2026-11-12-itl-994-997-job-run.md | 131 - .../2026-11-12-itl-996-spiral-tags-as-data.md | 81 - ...itl-1001-crash-recovery-flake-after-347.md | 110 - ...026-11-13-itl-1044-async-join-list-tags.md | 23 - ...2026-11-13-itl-1045-as-table-two-passes.md | 8 - ...-11-13-itl-1046-index-list-source-token.md | 25 - .../plans/2026-11-13-itl-1053-head-sort.md | 19 - ...026-11-13-itl-1060-readback-datagram-id.md | 9 - .../2026-11-13-itl-1065-interleaved-reads.md | 127 - ...13-itl-1066-side-effect-stream-as-table.md | 21 - .../2026-11-13-itl-1067-all-info-in-config.md | 16 - ...11-13-itl-1068-provisional-hash-warning.md | 103 - ...1-13-itl-1070-groupby-drops-non-by-tags.md | 76 - .../2026-11-13-itl-1074-test-inventory.md | 16 - .../plans/2026-11-13-itl-1075-async-hang.md | 155 - ...026-11-13-itl-1078-remove-resolve-field.md | 9 - ...026-11-13-itl-1080-sync-async-same-rows.md | 106 - .../2026-11-13-itl-1081-stream-run-view.md | 148 - ...-13-itl-1083-passthrough-operator-stale.md | 125 - ...2026-11-13-itl-1084-drop-select-columns.md | 72 - ...-13-itl-1091-side-effect-job-node-reads.md | 97 - ...-11-13-itl-1093-job-node-rows-after-run.md | 87 - ...26-11-13-itl-1095-node-current-run-view.md | 196 -- .../plans/2026-11-13-itl-1095-proposal.md | 196 -- ...026-11-13-itl-829-semijoin-nullable-key.md | 96 - .../2026-11-13-itl-879-async-error-policy.md | 160 - .../2026-11-13-itl-897-null-column-hash.md | 19 - ...026-11-13-itl-912-typed-zero-row-tables.md | 129 - .../2026-11-13-itl-940-tag-data-collision.md | 32 - ...6-11-14-itl-1054-pandas-refuse-at-write.md | 137 - ...26-11-14-itl-1079-s3-docstring-followup.md | 5 - .../2026-11-14-itl-1079-s3-tests-moto.md | 29 - ...-11-14-itl-1094-loaded-side-effect-node.md | 179 - .../2026-11-14-itl-1095-merge-main-itl-912.md | 50 - .../2026-11-14-itl-1111-definition-time.md | 66 - .../2026-11-14-itl-1112-bind-refuses-type.md | 127 - ...1-14-itl-1112-followup-merge-and-cr-392.md | 59 - ...6-11-14-itl-1121-node-view-review-fixes.md | 179 - ...2026-11-14-itl-1122-large-normalization.md | 127 - ...26-11-14-itl-1131-zero-row-column-order.md | 99 - ...6-11-14-itl-1137-type-hash-registration.md | 123 - ...11-14-itl-1138-loaded-slot-pending-type.md | 34 - .../2026-11-14-itl-611-set-of-primitives.md | 219 -- .../2026-11-14-plt-4864-refresh-for-v030.md | 84 - ...11-15-itl-1144-none-in-containers-guard.md | 191 -- .../plans/ITL-1000-derived-source-identity.md | 14 - .../plans/ITL-1082-job-database-rename.md | 17 - .../plans/ITL-1087-database-renames.md | 16 - .../plans/ITL-1102-run-checks-sources.md | 219 -- .../hotfix-loaded-side-effect-run-alone.md | 11 - ...47-ephemeral-database-side-effect-nodes.md | 13 - .../c10_held_node_run_keeps_last_run.py | 25 - .../c11_job_node_behind_outside_stream.py | 59 - .../itl_1121/c12_cr388_round2_traversal.py | 101 - .../c1_observer_reads_in_on_node_end.py | 59 - .../itl_1121/c2_operator_lazy_failure.py | 61 - .../itl_1121/c3_node_run_pending_writes.py | 102 - .../itl_1121/c4_summary_side_effect.py | 78 - .../itl_1121/c5_load_only_execute_failure.py | 84 - .../c6_load_only_over_side_effect_stream.py | 79 - .../c8_cr388_load_only_delivery_paths.py | 96 - .../itl_1121/c9_interrupted_log_commit.py | 57 - .../scripts/itl_1133_store_column_order.py | 70 - .../itl_897/claim_a_equality_vs_hash.py | 398 --- .../itl_897/claim_b_storage_roundtrip.py | 170 - .../scripts/itl_897/claim_d_golden_diff.py | 89 - .../scripts/itl_897/claim_d_migration.py | 168 - .../itl_897/claim_d_source_identity.py | 74 - .../itl_897/claim_f_table_value_paths.py | 68 - .../itl_897/claim_g_arrow_table_value.py | 91 - .../scripts/itl_897/claim_i_v1_descriptor.py | 199 -- superpowers/scripts/itl_897/prototype_v1.py | 264 -- .../2026-05-12-data-to-data-rename-design.md | 212 -- ...6-05-19-pipeline-job-source-spec-design.md | 485 --- .../specs/2026-05-20-polling-source-design.md | 686 ---- .../2026-05-21-networkx-replacement-design.md | 305 -- ...ipeline-pure-descriptor-refactor-design.md | 519 --- ...23-pipeline-pipelinejob-refactor-design.md | 489 --- ...17-pipeline-job-execution-graph-cleanup.md | 234 -- ...26-05-29-pipeline-dag-public-api-design.md | 241 -- ...-dataclass-arrow-struct-encoding-design.md | 335 -- ...unction-node-fetch-join-refactor-design.md | 162 - ...26-06-03-pipeline-test-migration-design.md | 217 -- ...26-06-04-function-node-empty-schema-fix.md | 368 -- ...16-source-node-column-config-forwarding.md | 185 - ...-08-eng-376-function-node-pod-interface.md | 60 - ...-eng-389-empty-container-type-inference.md | 138 - ...ng-571-tag-columns-normalization-design.md | 230 -- ...-06-08-plt-964-persistent-config-design.md | 293 -- ...6-06-11-plt-1162-uuid-arrow-type-design.md | 213 -- ...6-06-11-type-specific-pod-access-design.md | 109 - ...6-06-13-datetime-arrow-timestamp-design.md | 173 - ...xtension-type-converter-protocol-design.md | 89 - ...26-06-14-extension-type-registry-design.md | 277 -- ...026-06-14-plt-1654-schema-walker-design.md | 202 -- ...26-06-14-plt-1655-database-hooks-design.md | 387 --- ...4-plt-1656-builtin-logical-types-design.md | 308 -- ...26-06-14-plt-1668-logical-type-redesign.md | 271 -- ...-write-side-logical-type-factory-design.md | 510 --- ...t-1705-type-registration-spine-refactor.md | 362 -- ...ister-python-class-storage-type-cleanup.md | 253 -- ...17-pydantic-logical-type-factory-design.md | 231 -- ...-factories-into-default-registry-design.md | 182 - ...26-06-21-schema-metadata-cleaner-design.md | 191 -- ...-pyspiral-upgrade-fix-ci-header-signing.md | 119 - ...type-roundtrip-integration-tests-design.md | 228 -- ...lt-1660-hard-cut-extension-type-hashing.md | 628 ---- ...antic-dataclass-pipeline-columns-design.md | 194 -- .../2026-06-28-pick-index-operators-design.md | 447 --- ...06-29-itl-45-datetime-tag-values-design.md | 188 -- ...2026-06-29-linear-release-system-design.md | 176 - .../specs/2026-06-29-op-file-type-design.md | 254 -- ...026-06-29-typing-literal-support-design.md | 156 - ...6-29-union-typed-function-inputs-design.md | 128 - ...-06-30-itl-451-op-directory-type-design.md | 373 -- ...ignature-hash-order-independence-design.md | 178 - ...-numpy-ndarray-native-value-type-design.md | 244 -- .../2026-06-30-release-workflow-design.md | 191 -- .../2026-07-01-pandas-native-types-design.md | 212 -- ...-07-02-itl-472-file-hash-caching-design.md | 343 -- .../2026-07-03-itl-489-warm-cache-design.md | 174 - ...-03-itl-493-pathlike-conformance-design.md | 125 - ...-07-03-itl-concurrent-warm-cache-design.md | 220 -- ...0-populate-hash-cache-ergonomics-design.md | 288 -- ...07-dict-source-auto-registration-design.md | 101 - ...itl-474-opfile-url-form-identity-design.md | 117 - ...07-ephemeral-function-pod-output-design.md | 479 --- ...class-dataframe-conversion-tests-design.md | 249 -- ...aldb-arrow-metadata-preservation-design.md | 372 -- ...-508-indexed-entry-id-versioning-design.md | 219 -- ...8-itl-511-directory-hash-caching-design.md | 144 - ...tl-513-stale-entry-overwrite-resolution.md | 76 - ...-08-node-config-pod-config-split-design.md | 280 -- ...d-extension-spikeinterface-split-design.md | 380 --- ...7-09-itl-516-unify-async-execute-design.md | 255 -- ...itl-519-hash-cacher-write-guards-design.md | 252 -- ...-10-itl-520-postgres-hash-cacher-design.md | 269 -- ...6-07-13-itl-522-match-mtime-flag-design.md | 321 -- ...2026-07-14-itl-523-post-run-hook-design.md | 288 -- ...6-07-14-itl-524-side-effect-pods-design.md | 978 ------ ...-14-itl-525-side-effect-pod-impl-design.md | 666 ---- ...empty-data-ephemeral-propagation-design.md | 347 -- ...itl-532-side-effect-function-pod-design.md | 626 ---- ...content-hash-preimage-redundancy-design.md | 452 --- ...26-07-17-ctx-arg-and-ray-opts-hardening.md | 130 - ...-535-schema-versioning-migration-design.md | 518 --- ...-07-21-hash-columns-large-binary-design.md | 189 -- .../2026-07-21-itl-529-hash-audit-design.md | 358 -- ...-itl-545-release-branch-argument-design.md | 110 - ...ostrunpayload-invocation-context-design.md | 190 -- ...26-07-22-empty-table-nullability-design.md | 143 - ...sion-deserializer-storage-family-design.md | 141 - ...603-directory-exclude-by-pattern-design.md | 160 - ...6-08-07-npipe-204-batch-group-by-design.md | 363 -- ...1-itl-173-list-set-logical-types-design.md | 415 --- ...20-github-actions-node24-upgrade-design.md | 65 - ...source-pipelinejob-async-delegation-fix.md | 122 - ...6-08-22-itl-627-list-extension-metadata.md | 348 -- ...polling-source-combine-content-hash-fix.md | 131 - ...7-pollingsource-async-mode-guard-design.md | 492 --- ...g-952-polling-source-zero-row-batch-fix.md | 183 - ...8-stable-type-annotation-hashing-design.md | 215 -- ...ema-hashing-arrow-representation-design.md | 289 -- ...1-09-itl-817-content-derived-record-ids.md | 97 - .../specs/2026-11-10-array-format-v0.3.md | 394 --- .../2026-11-10-user-class-identity-v0.3.md | 420 --- ...-itl-186-spiral-streaming-source-design.md | 239 -- ...-11-itl-828-polars-expr-identity-design.md | 141 - ...2026-11-11-itl-897-pandas-hash-identity.md | 524 --- ...-11-11-itl-938-versioning-system-design.md | 1862 ---------- ...1-itl-939-tag-and-data-same-name-design.md | 912 ----- .../2026-11-11-public-release-readiness.md | 481 --- .../specs/2026-11-11-torch-tensor-v0.3.md | 274 -- ...-itl-186-streaming-spiral-source-design.md | 367 -- .../2026-11-12-itl-186-what-a-job-keeps.md | 41 - .../2026-11-12-itl-972-user-class-kinds.md | 447 --- ...1-12-itl-981-library-value-types-design.md | 355 -- ...-itl-991-stream-head-and-preview-design.md | 409 --- .../2026-11-13-itl-1074-test-inventory.md | 159 - ...-11-14-itl-611-set-of-primitives-design.md | 182 - .../2026-11-15-itl-1154-golden-additions.tsv | 95 - .../specs/2026-11-15-itl-1154-golden-moves.md | 69 - .../2026-11-15-itl-1154-golden-moves.tsv | 849 ----- ...2026-11-15-itl-1154-hash-site-inventory.md | 177 - .../2026-11-15-itl-1154-row-identity-note.md | 54 - .../2026-11-16-itl-1154-key-policy-moves.md | 73 - .../2026-11-16-itl-1154-key-policy-moves.tsv | 108 - superpowers/specs/itl-1034-sweep/README.md | 22 - .../datagram_uuid_after_readback.py | 63 - .../interleaved_reads_recompute.py | 39 - .../itl1043_loaded_operator_cache.py | 122 - .../itl1045_two_pass_as_table.py | 47 - .../itl-1034-sweep/itl1046_index_token.py | 38 - .../itl1047_same_stream_twice.py | 43 - .../itl-1034-sweep/partial_read_then_head.py | 31 - .../side_effect_stream_as_table.py | 39 - superpowers/specs/itl-1040/README.md | 28 - superpowers/specs/itl-1040/measure.py | 248 -- .../specs/itl-1042/rowless_head_columns.py | 152 - superpowers/specs/itl-1043/FINDINGS.md | 120 - superpowers/specs/itl-1043/q1_repro.py | 89 - .../specs/itl-1043/q2_where_rows_go.py | 63 - superpowers/specs/itl-1043/q3_schemas.py | 95 - superpowers/specs/itl-1043/q4_blast_radius.py | 177 - .../specs/itl-1043/q4b_from_pipeline.py | 85 - .../specs/itl-1043/q5_nullability_hash.py | 52 - superpowers/specs/itl-1043/q6_live_replay.py | 68 - .../as_table_after_sync_vs_async_run.py | 52 - .../d2_restriction_identity.py | 140 - superpowers/specs/itl-828-experiments/exp1.py | 69 - superpowers/specs/itl-828-experiments/exp2.py | 71 - .../specs/itl-892-exit-hang/FINDINGS.md | 508 --- .../check_native_fs_equivalence.py | 121 - .../specs/itl-892-exit-hang/harness.py | 150 - .../linux/round1-py3.11/results.jsonl | 13 - .../linux/round1-py3.12/results.jsonl | 13 - .../linux/round1-py3.13/results.jsonl | 13 - .../linux_op=to_table_py3.13_locked.txt | 348 -- .../linux/round2-py3.11/results.jsonl | 14 - .../linux/round2-py3.12/results.jsonl | 14 - .../linux/round2-py3.13/results.jsonl | 14 - ...od_plain-read_patch=none_py3.13_locked.txt | 276 -- .../native_fs_pytest_plugin.py | 11 - .../regression_check_proto.py | 56 - .../specs/itl-892-exit-hang/repro_orcapod.py | 202 -- .../specs/itl-892-exit-hang/repro_pure.py | 216 -- .../specs/itl-892-exit-hang/results.jsonl | 82 - .../specs/itl-892-exit-hang/run_matrix.sh | 75 - .../dataset_scan_py3.13_pa23.0.1_dl1.0.2.txt | 929 ----- .../samples/op=dataset_scan_py3.12_locked.txt | 894 ----- .../orcapod_plain_read_hang_manual.txt | 1042 ------ .../specs/itl-892-exit-hang/summarize.py | 78 - .../01_pyarrow_minimal.py | 163 - .../itl-895-float-to-int/02_call_chain.py | 138 - .../itl-895-float-to-int/03_split_datagram.py | 69 - .../itl-895-float-to-int/04_input_sites.py | 186 - .../itl-895-float-to-int/05_pod_matrix.py | 146 - .../06_released_versions.py | 31 - .../itl-895-float-to-int/07_blast_radius.py | 79 - .../itl-895-float-to-int/08_none_output.py | 93 - .../itl-895-float-to-int/09_warning_dedup.py | 82 - .../specs/itl-895-float-to-int/FINDINGS.md | 328 -- .../specs/itl-895-float-to-int/_tracer.py | 167 - superpowers/specs/itl-926-live/FINDINGS.md | 232 -- .../specs/itl-926-live/error_messages.py | 57 - .../specs/itl-926-live/inspect_tables.py | 79 - superpowers/specs/itl-926-live/live_verify.py | 727 ---- .../proposals/c3b_differential.py | 68 - .../c1_nullable_columns_refuse_plain_pods.py | 52 - .../repros/c2_fixed_size_list_cannot_hash.py | 24 - .../itl-926-live/repros/c3_source_memory.py | 46 - .../repros/h1_timestamp_tag_becomes_utc.py | 44 - .../repros/h2_positional_record_ids.py | 44 - .../h4_get_all_records_delta_after_run.py | 52 - .../source_id_collision_across_projects.py | 80 - .../specs/itl-926-live/scan_cost_probe.py | 65 - superpowers/specs/itl-926-live/size_probe.py | 119 - .../specs/itl-936-live/live_nullability.py | 535 --- .../exp01_same_name_today.out | 56 - .../exp01_same_name_today.py | 200 -- .../exp02_workarounds_today.out | 69 - .../exp02_workarounds_today.py | 226 -- .../exp03_copy_tag_prototype.out | 53 - .../exp03_copy_tag_prototype.py | 251 -- .../exp04_foreign_key_promotion.out | 13 - .../exp04_foreign_key_promotion.py | 84 - .../exp05_hash_comparison.out | 24 - .../exp05_hash_comparison.py | 121 - .../exp06_derived_source_today.out | 46 - .../exp06_derived_source_today.py | 156 - .../exp07_derived_source_prototype.out | 58 - .../exp07_derived_source_prototype.py | 334 -- .../exp08_group_by_tags_into_data.out | 17 - .../exp08_group_by_tags_into_data.py | 59 - .../spiraldb_derived_source_check.py | 73 - tests/test_arrays/test_array_value_format.py | 1 - .../test_node_view_review_fixes.py | 2 +- 483 files changed, 413 insertions(+), 162585 deletions(-) rename {superpowers/scripts => scripts}/hash_audit_example.py (99%) delete mode 100644 superpowers/plans/2026-05-12-data-to-data-rename.md delete mode 100644 superpowers/plans/2026-05-20-pipeline-job-source-spec.md delete mode 100644 superpowers/plans/2026-05-20-polling-source.md delete mode 100644 superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md delete mode 100644 superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md delete mode 100644 superpowers/plans/2026-05-23-eng-515-pipeline-pipelinejob-refactor.md delete mode 100644 superpowers/plans/2026-05-23-rename-config-to-orcapodconfig.md delete mode 100644 superpowers/plans/2026-05-26-eng-517-pipeline-job-execution-graph-cleanup.md delete mode 100644 superpowers/plans/2026-05-29-pipeline-dag-public-api.md delete mode 100644 superpowers/plans/2026-06-02-dataclass-arrow-struct-encoding.md delete mode 100644 superpowers/plans/2026-06-02-eng-377-function-node-fetch-join-refactor.md delete mode 100644 superpowers/plans/2026-06-03-pipeline-test-migration.md delete mode 100644 superpowers/plans/2026-06-05-eng-516-source-node-column-config-forwarding.md delete mode 100644 superpowers/plans/2026-06-08-eng-376-result-database-rename.md delete mode 100644 superpowers/plans/2026-06-08-eng-389-empty-container-type-inference.md delete mode 100644 superpowers/plans/2026-06-08-eng-571-tag-columns-normalization.md delete mode 100644 superpowers/plans/2026-06-08-plt-964-persistent-config.md delete mode 100644 superpowers/plans/2026-06-11-plt-1162-uuid-arrow-type.md delete mode 100644 superpowers/plans/2026-06-11-type-specific-pod-access.md delete mode 100644 superpowers/plans/2026-06-13-datetime-arrow-timestamp.md delete mode 100644 superpowers/plans/2026-06-13-plt-1652-extension-type-converter-protocol.md delete mode 100644 superpowers/plans/2026-06-14-extension-type-registry.md delete mode 100644 superpowers/plans/2026-06-14-plt-1654-schema-walker.md delete mode 100644 superpowers/plans/2026-06-14-plt-1655-database-hooks.md delete mode 100644 superpowers/plans/2026-06-14-plt-1656-builtin-logical-types.md delete mode 100644 superpowers/plans/2026-06-14-plt-1668-logical-type-redesign.md delete mode 100644 superpowers/plans/2026-06-15-plt-1670-orcapod-namespace-builtin-types.md delete mode 100644 superpowers/plans/2026-06-15-plt-1672-write-side-logical-type-factory.md delete mode 100644 superpowers/plans/2026-06-16-plt-1705-type-registration-spine-refactor.md delete mode 100644 superpowers/plans/2026-06-17-plt-1720-register-python-class-storage-type-cleanup.md delete mode 100644 superpowers/plans/2026-06-17-plt-1731-pydantic-logical-type-factory.md delete mode 100644 superpowers/plans/2026-06-18-plt-1701-wire-factories-into-default-registry.md delete mode 100644 superpowers/plans/2026-06-21-plt-1737-schema-metadata-cleaner.md delete mode 100644 superpowers/plans/2026-06-23-plt-1659-extension-type-roundtrip-integration-tests.md delete mode 100644 superpowers/plans/2026-06-23-pyspiral-upgrade-plt-1773.md delete mode 100644 superpowers/plans/2026-06-24-plt-1660-hard-cut-extension-type-hashing.md delete mode 100644 superpowers/plans/2026-06-24-rename-semantic-hasher-to-handler.md delete mode 100644 superpowers/plans/2026-06-25-plt-1663-merge-extension-type-system-into-main.md delete mode 100644 superpowers/plans/2026-06-28-pick-index-operators.md delete mode 100644 superpowers/plans/2026-06-29-itl-45-datetime-tag-values.md delete mode 100644 superpowers/plans/2026-06-29-linear-release-system.md delete mode 100644 superpowers/plans/2026-06-29-typing-literal-support.md delete mode 100644 superpowers/plans/2026-06-30-itl-453-union-signature-hash-order-independence.md delete mode 100644 superpowers/plans/2026-06-30-op-file-type.md delete mode 100644 superpowers/plans/2026-06-30-release-workflow.md delete mode 100644 superpowers/plans/2026-06-30-union-typed-function-inputs.md delete mode 100644 superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md delete mode 100644 superpowers/plans/2026-07-01-itl-460-numpy-ndarray-native-value-type.md delete mode 100644 superpowers/plans/2026-07-01-pandas-native-types.md delete mode 100644 superpowers/plans/2026-07-02-itl-472-file-hash-caching.md delete mode 100644 superpowers/plans/2026-07-02-si-sorting-logical-type.md delete mode 100644 superpowers/plans/2026-07-02-spikeinterface-motion.md delete mode 100644 superpowers/plans/2026-07-03-itl-489-warm-cache.md delete mode 100644 superpowers/plans/2026-07-03-itl-493-pathlike-conformance.md delete mode 100644 superpowers/plans/2026-07-03-itl-concurrent-warm-cache.md delete mode 100644 superpowers/plans/2026-07-03-sorting-analyzer-logical-type.md delete mode 100644 superpowers/plans/2026-07-05-itl-500-populate-hash-cache-ergonomics.md delete mode 100644 superpowers/plans/2026-07-07-dict-source-auto-registration.md delete mode 100644 superpowers/plans/2026-07-07-itl-142-pydantic-missing-tests.md delete mode 100644 superpowers/plans/2026-07-07-itl-507-ephemeral-function-pod-output.md delete mode 100644 superpowers/plans/2026-07-08-itl-471-spiraldb-arrow-metadata-preservation.md delete mode 100644 superpowers/plans/2026-07-08-itl-508-indexed-entry-id-versioning.md delete mode 100644 superpowers/plans/2026-07-08-itl-511-directory-hash-caching.md delete mode 100644 superpowers/plans/2026-07-09-itl-473-orcapod-extension-spikeinterface.md delete mode 100644 superpowers/plans/2026-07-09-node-config-pod-config-split.md delete mode 100644 superpowers/plans/2026-07-10-itl-516-unify-async-execute.md delete mode 100644 superpowers/plans/2026-07-10-itl-519-hash-cacher-write-guards.md delete mode 100644 superpowers/plans/2026-07-10-itl-520-postgres-hash-cacher.md delete mode 100644 superpowers/plans/2026-07-14-itl-523-post-run-hook.md delete mode 100644 superpowers/plans/2026-07-14-itl-525-side-effect-pod.md delete mode 100644 superpowers/plans/2026-07-16-empty-data-ephemeral-propagation.md delete mode 100644 superpowers/plans/2026-07-16-itl-532-side-effect-function-pod.md delete mode 100644 superpowers/plans/2026-07-16-itl-532-unified-function-pod.md delete mode 100644 superpowers/plans/2026-07-17-ctx-arg-and-ray-opts-hardening.md delete mode 100644 superpowers/plans/2026-07-17-itl-535-schema-versioning-migration.md delete mode 100644 superpowers/plans/2026-07-21-itl-533-remove-node-content-hash-preimage.md delete mode 100644 superpowers/plans/2026-07-21-itl-545-release-branch-argument.md delete mode 100644 superpowers/plans/2026-07-21-postrunpayload-invocation-context.md delete mode 100644 superpowers/plans/2026-07-22-empty-table-nullability.md delete mode 100644 superpowers/plans/2026-08-05-itl-603-directory-exclude-by-pattern.md delete mode 100644 superpowers/plans/2026-08-07-npipe-204-groupby-operator.md delete mode 100644 superpowers/plans/2026-08-11-itl-173-list-set-logical-types.md delete mode 100644 superpowers/plans/2026-08-20-github-actions-node24-upgrade.md delete mode 100644 superpowers/plans/2026-08-20-itl-615-pollingsource-pipelinejob-async-delegation-fix.md delete mode 100644 superpowers/plans/2026-08-22-itl-627-list-extension-metadata.md delete mode 100644 superpowers/plans/2026-08-25-itl-616-polling-source-combine-content-hash-fix.md delete mode 100644 superpowers/plans/2026-08-25-itl-617-pollingsource-concurrent-cursor-safety.md delete mode 100644 superpowers/plans/2026-08-27-eng-952-polling-source-zero-row-batch-fix.md delete mode 100644 superpowers/plans/2026-09-02-itl-638-stable-type-annotation-hashing.md delete mode 100644 superpowers/plans/2026-09-03-itl-639-schema-hashing.md delete mode 100644 superpowers/plans/2026-11-09-docs-gap-report.md delete mode 100644 superpowers/plans/2026-11-09-itl-134-remove-pygraphviz.md delete mode 100644 superpowers/plans/2026-11-09-itl-810-v0.2-release-prep.md delete mode 100644 superpowers/plans/2026-11-09-itl-813-mergejoin-empty-input.md delete mode 100644 superpowers/plans/2026-11-09-itl-816-iter-data-chunk-layouts.md delete mode 100644 superpowers/plans/2026-11-09-itl-817-system-tag-run-stability.md delete mode 100644 superpowers/plans/2026-11-09-itl-825-python-313-canonical-type-names.md delete mode 100644 superpowers/plans/2026-11-09-itl-826-org-required-ci.md delete mode 100644 superpowers/plans/2026-11-09-itl-827-packaging-cleanup.md delete mode 100644 superpowers/plans/2026-11-09-itl-832-mergejoin-arrow-types.md delete mode 100644 superpowers/plans/2026-11-09-itl-834-index-pick-arrow-types.md delete mode 100644 superpowers/plans/2026-11-09-plt-4816-docs-audit.md delete mode 100644 superpowers/plans/2026-11-09-pr-272-review-revision.md delete mode 100644 superpowers/plans/2026-11-09-pr-275-coderabbit-review.md delete mode 100644 superpowers/plans/2026-11-09-v0.2-release-checklist.md delete mode 100644 superpowers/plans/2026-11-09-workshop-facilitator-notes.md delete mode 100644 superpowers/plans/2026-11-09-workshop-tutorial.md delete mode 100644 superpowers/plans/2026-11-10-itl-578-golden-merge.md delete mode 100644 superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md delete mode 100644 superpowers/plans/2026-11-10-itl-578-polars-hash-labels.md delete mode 100644 superpowers/plans/2026-11-10-itl-626-golden-generator-timeout.md delete mode 100644 superpowers/plans/2026-11-10-itl-626-golden-hash-regression.md delete mode 100644 superpowers/plans/2026-11-10-itl-810-finalize-v0.2-release-docs.md delete mode 100644 superpowers/plans/2026-11-10-itl-810-refresh-pr-269-public-release.md delete mode 100644 superpowers/plans/2026-11-10-itl-813-832-mergejoin-arrow-types-rework.md delete mode 100644 superpowers/plans/2026-11-10-itl-837-join-async-arrow-types.md delete mode 100644 superpowers/plans/2026-11-10-itl-845-per-row-source-info.md delete mode 100644 superpowers/plans/2026-11-10-itl-847-semijoin-nested-columns.md delete mode 100644 superpowers/plans/2026-11-10-itl-849-chained-aggregation-source-info.md delete mode 100644 superpowers/plans/2026-11-10-itl-854-join-async-meta-columns.md delete mode 100644 superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md delete mode 100644 superpowers/plans/2026-11-10-itl-875-mit-license.md delete mode 100644 superpowers/plans/2026-11-10-itl-878-store-write-failure.md delete mode 100644 superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md delete mode 100644 superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md delete mode 100644 superpowers/plans/2026-11-10-itl-884-delta-read-failures.md delete mode 100644 superpowers/plans/2026-11-10-itl-888-array-format.md delete mode 100644 superpowers/plans/2026-11-10-itl-892-exit-hang.md delete mode 100644 superpowers/plans/2026-11-10-pr-281-public-call-tests.md delete mode 100644 superpowers/plans/2026-11-10-pr-282-merge-main-itl-849.md delete mode 100644 superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md delete mode 100644 superpowers/plans/2026-11-10-pr280-review-shared-arrow-type.md delete mode 100644 superpowers/plans/2026-11-11-itl-578-merge-main-semijoin.md delete mode 100644 superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md delete mode 100644 superpowers/plans/2026-11-11-itl-828-polars-expr-identity.md delete mode 100644 superpowers/plans/2026-11-11-itl-864-stack-on-290.md delete mode 100644 superpowers/plans/2026-11-11-itl-871-torch-tensor.md delete mode 100644 superpowers/plans/2026-11-11-itl-875-public-release-readiness.md delete mode 100644 superpowers/plans/2026-11-11-itl-881-field-order-identity.md delete mode 100644 superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md delete mode 100644 superpowers/plans/2026-11-11-itl-895-decisions.md delete mode 100644 superpowers/plans/2026-11-11-itl-895-float-to-int.md delete mode 100644 superpowers/plans/2026-11-11-itl-896-async-operator-readback.md delete mode 100644 superpowers/plans/2026-11-11-itl-897-pandas-hash-identity.md delete mode 100644 superpowers/plans/2026-11-11-itl-904-loaded-job-readback.md delete mode 100644 superpowers/plans/2026-11-11-itl-905-dict-order-hash.md delete mode 100644 superpowers/plans/2026-11-11-itl-910-adopt-starfix-05.md delete mode 100644 superpowers/plans/2026-11-11-itl-910-followup-ignore-old-label-rows.md delete mode 100644 superpowers/plans/2026-11-11-itl-919-delta-batched-commits.md delete mode 100644 superpowers/plans/2026-11-11-itl-919-in-memory-store-linear.md delete mode 100644 superpowers/plans/2026-11-11-itl-926-spiral-source-reverify.md delete mode 100644 superpowers/plans/2026-11-11-itl-935-logical-type-identifiers.md delete mode 100644 superpowers/plans/2026-11-11-itl-936-spiral-nullability-record.md delete mode 100644 superpowers/plans/2026-11-11-itl-938-post-session-decisions.md delete mode 100644 superpowers/plans/2026-11-11-itl-939-derived-source-tags-as-data.md delete mode 100644 superpowers/plans/2026-11-11-itl-939-tag-and-data-same-name-spike.md delete mode 100644 superpowers/plans/2026-11-11-itl-942-wrong-hash-fixes.md delete mode 100644 superpowers/plans/2026-11-11-itl-943-drop-nulls.md delete mode 100644 superpowers/plans/2026-11-11-itl-944-data-context-v1.md delete mode 100644 superpowers/plans/2026-11-11-itl-945-hash-label-rule.md delete mode 100644 superpowers/plans/2026-11-11-itl-953-function-defaults-semantic-hash.md delete mode 100644 superpowers/plans/2026-11-11-itl-954-prefix-scan.md delete mode 100644 superpowers/plans/2026-11-11-itl-955-groupby-tags-stay-tags.md delete mode 100644 superpowers/plans/2026-11-11-itl-957-default-source-ids.md delete mode 100644 superpowers/plans/2026-11-11-itl-961-changelog-fragments.md delete mode 100644 superpowers/plans/2026-11-11-itl-967-hash-step-combined.md delete mode 100644 superpowers/plans/2026-11-11-itl-968-design-issues-one-file-per-entry.md delete mode 100644 superpowers/plans/2026-11-12-itl-1001-flaky-async-run.md delete mode 100644 superpowers/plans/2026-11-12-itl-1007-public-release.md delete mode 100644 superpowers/plans/2026-11-12-itl-1026-orcapod-version-stamp.md delete mode 100644 superpowers/plans/2026-11-12-itl-1027-map-keep-default.md delete mode 100644 superpowers/plans/2026-11-12-itl-1034-row-alignment-suite.md delete mode 100644 superpowers/plans/2026-11-12-itl-1039-resumed-iteration.md delete mode 100644 superpowers/plans/2026-11-12-itl-1040-dropped-column-source-info.md delete mode 100644 superpowers/plans/2026-11-12-itl-1042-head-after-async.md delete mode 100644 superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md delete mode 100644 superpowers/plans/2026-11-12-itl-186-notebook-preview.md delete mode 100644 superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md delete mode 100644 superpowers/plans/2026-11-12-itl-578-refresh-onto-v03-hash-step.md delete mode 100644 superpowers/plans/2026-11-12-itl-807-function-pod-auto-select.md delete mode 100644 superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md delete mode 100644 superpowers/plans/2026-11-12-itl-871-refresh.md delete mode 100644 superpowers/plans/2026-11-12-itl-895-refresh.md delete mode 100644 superpowers/plans/2026-11-12-itl-895-review-round-c7f53bc0.md delete mode 100644 superpowers/plans/2026-11-12-itl-897-refresh-onto-v03-hash-step.md delete mode 100644 superpowers/plans/2026-11-12-itl-904-refresh.md delete mode 100644 superpowers/plans/2026-11-12-itl-905-refresh-against-main.md delete mode 100644 superpowers/plans/2026-11-12-itl-906-async-system-tag-names.md delete mode 100644 superpowers/plans/2026-11-12-itl-928-node-tag-types.md delete mode 100644 superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md delete mode 100644 superpowers/plans/2026-11-12-itl-967-golden-entries.md delete mode 100644 superpowers/plans/2026-11-12-itl-972-user-class-kinds.md delete mode 100644 superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md delete mode 100644 superpowers/plans/2026-11-12-itl-981-library-value-types.md delete mode 100644 superpowers/plans/2026-11-12-itl-984-delta-second-commit.md delete mode 100644 superpowers/plans/2026-11-12-itl-988-cached-pod-job.md delete mode 100644 superpowers/plans/2026-11-12-itl-991-stream-head.md delete mode 100644 superpowers/plans/2026-11-12-itl-994-997-job-run.md delete mode 100644 superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md delete mode 100644 superpowers/plans/2026-11-13-itl-1001-crash-recovery-flake-after-347.md delete mode 100644 superpowers/plans/2026-11-13-itl-1044-async-join-list-tags.md delete mode 100644 superpowers/plans/2026-11-13-itl-1045-as-table-two-passes.md delete mode 100644 superpowers/plans/2026-11-13-itl-1046-index-list-source-token.md delete mode 100644 superpowers/plans/2026-11-13-itl-1053-head-sort.md delete mode 100644 superpowers/plans/2026-11-13-itl-1060-readback-datagram-id.md delete mode 100644 superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md delete mode 100644 superpowers/plans/2026-11-13-itl-1066-side-effect-stream-as-table.md delete mode 100644 superpowers/plans/2026-11-13-itl-1067-all-info-in-config.md delete mode 100644 superpowers/plans/2026-11-13-itl-1068-provisional-hash-warning.md delete mode 100644 superpowers/plans/2026-11-13-itl-1070-groupby-drops-non-by-tags.md delete mode 100644 superpowers/plans/2026-11-13-itl-1074-test-inventory.md delete mode 100644 superpowers/plans/2026-11-13-itl-1075-async-hang.md delete mode 100644 superpowers/plans/2026-11-13-itl-1078-remove-resolve-field.md delete mode 100644 superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md delete mode 100644 superpowers/plans/2026-11-13-itl-1081-stream-run-view.md delete mode 100644 superpowers/plans/2026-11-13-itl-1083-passthrough-operator-stale.md delete mode 100644 superpowers/plans/2026-11-13-itl-1084-drop-select-columns.md delete mode 100644 superpowers/plans/2026-11-13-itl-1091-side-effect-job-node-reads.md delete mode 100644 superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md delete mode 100644 superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md delete mode 100644 superpowers/plans/2026-11-13-itl-1095-proposal.md delete mode 100644 superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md delete mode 100644 superpowers/plans/2026-11-13-itl-879-async-error-policy.md delete mode 100644 superpowers/plans/2026-11-13-itl-897-null-column-hash.md delete mode 100644 superpowers/plans/2026-11-13-itl-912-typed-zero-row-tables.md delete mode 100644 superpowers/plans/2026-11-13-itl-940-tag-data-collision.md delete mode 100644 superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md delete mode 100644 superpowers/plans/2026-11-14-itl-1079-s3-docstring-followup.md delete mode 100644 superpowers/plans/2026-11-14-itl-1079-s3-tests-moto.md delete mode 100644 superpowers/plans/2026-11-14-itl-1094-loaded-side-effect-node.md delete mode 100644 superpowers/plans/2026-11-14-itl-1095-merge-main-itl-912.md delete mode 100644 superpowers/plans/2026-11-14-itl-1111-definition-time.md delete mode 100644 superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md delete mode 100644 superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md delete mode 100644 superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md delete mode 100644 superpowers/plans/2026-11-14-itl-1122-large-normalization.md delete mode 100644 superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md delete mode 100644 superpowers/plans/2026-11-14-itl-1137-type-hash-registration.md delete mode 100644 superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md delete mode 100644 superpowers/plans/2026-11-14-itl-611-set-of-primitives.md delete mode 100644 superpowers/plans/2026-11-14-plt-4864-refresh-for-v030.md delete mode 100644 superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md delete mode 100644 superpowers/plans/ITL-1000-derived-source-identity.md delete mode 100644 superpowers/plans/ITL-1082-job-database-rename.md delete mode 100644 superpowers/plans/ITL-1087-database-renames.md delete mode 100644 superpowers/plans/ITL-1102-run-checks-sources.md delete mode 100644 superpowers/plans/hotfix-loaded-side-effect-run-alone.md delete mode 100644 superpowers/plans/itl-1147-ephemeral-database-side-effect-nodes.md delete mode 100644 superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py delete mode 100644 superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py delete mode 100644 superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py delete mode 100644 superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py delete mode 100644 superpowers/scripts/itl_1121/c2_operator_lazy_failure.py delete mode 100644 superpowers/scripts/itl_1121/c3_node_run_pending_writes.py delete mode 100644 superpowers/scripts/itl_1121/c4_summary_side_effect.py delete mode 100644 superpowers/scripts/itl_1121/c5_load_only_execute_failure.py delete mode 100644 superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py delete mode 100644 superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py delete mode 100644 superpowers/scripts/itl_1121/c9_interrupted_log_commit.py delete mode 100644 superpowers/scripts/itl_1133_store_column_order.py delete mode 100644 superpowers/scripts/itl_897/claim_a_equality_vs_hash.py delete mode 100644 superpowers/scripts/itl_897/claim_b_storage_roundtrip.py delete mode 100644 superpowers/scripts/itl_897/claim_d_golden_diff.py delete mode 100644 superpowers/scripts/itl_897/claim_d_migration.py delete mode 100644 superpowers/scripts/itl_897/claim_d_source_identity.py delete mode 100644 superpowers/scripts/itl_897/claim_f_table_value_paths.py delete mode 100644 superpowers/scripts/itl_897/claim_g_arrow_table_value.py delete mode 100644 superpowers/scripts/itl_897/claim_i_v1_descriptor.py delete mode 100644 superpowers/scripts/itl_897/prototype_v1.py delete mode 100644 superpowers/specs/2026-05-12-data-to-data-rename-design.md delete mode 100644 superpowers/specs/2026-05-19-pipeline-job-source-spec-design.md delete mode 100644 superpowers/specs/2026-05-20-polling-source-design.md delete mode 100644 superpowers/specs/2026-05-21-networkx-replacement-design.md delete mode 100644 superpowers/specs/2026-05-21-pipeline-pure-descriptor-refactor-design.md delete mode 100644 superpowers/specs/2026-05-23-pipeline-pipelinejob-refactor-design.md delete mode 100644 superpowers/specs/2026-05-26-eng-517-pipeline-job-execution-graph-cleanup.md delete mode 100644 superpowers/specs/2026-05-29-pipeline-dag-public-api-design.md delete mode 100644 superpowers/specs/2026-06-02-dataclass-arrow-struct-encoding-design.md delete mode 100644 superpowers/specs/2026-06-02-function-node-fetch-join-refactor-design.md delete mode 100644 superpowers/specs/2026-06-03-pipeline-test-migration-design.md delete mode 100644 superpowers/specs/2026-06-04-function-node-empty-schema-fix.md delete mode 100644 superpowers/specs/2026-06-05-eng-516-source-node-column-config-forwarding.md delete mode 100644 superpowers/specs/2026-06-08-eng-376-function-node-pod-interface.md delete mode 100644 superpowers/specs/2026-06-08-eng-389-empty-container-type-inference.md delete mode 100644 superpowers/specs/2026-06-08-eng-571-tag-columns-normalization-design.md delete mode 100644 superpowers/specs/2026-06-08-plt-964-persistent-config-design.md delete mode 100644 superpowers/specs/2026-06-11-plt-1162-uuid-arrow-type-design.md delete mode 100644 superpowers/specs/2026-06-11-type-specific-pod-access-design.md delete mode 100644 superpowers/specs/2026-06-13-datetime-arrow-timestamp-design.md delete mode 100644 superpowers/specs/2026-06-13-plt-1652-extension-type-converter-protocol-design.md delete mode 100644 superpowers/specs/2026-06-14-extension-type-registry-design.md delete mode 100644 superpowers/specs/2026-06-14-plt-1654-schema-walker-design.md delete mode 100644 superpowers/specs/2026-06-14-plt-1655-database-hooks-design.md delete mode 100644 superpowers/specs/2026-06-14-plt-1656-builtin-logical-types-design.md delete mode 100644 superpowers/specs/2026-06-14-plt-1668-logical-type-redesign.md delete mode 100644 superpowers/specs/2026-06-15-plt-1672-write-side-logical-type-factory-design.md delete mode 100644 superpowers/specs/2026-06-16-plt-1705-type-registration-spine-refactor.md delete mode 100644 superpowers/specs/2026-06-17-plt-1720-register-python-class-storage-type-cleanup.md delete mode 100644 superpowers/specs/2026-06-17-pydantic-logical-type-factory-design.md delete mode 100644 superpowers/specs/2026-06-18-plt-1701-wire-factories-into-default-registry-design.md delete mode 100644 superpowers/specs/2026-06-21-schema-metadata-cleaner-design.md delete mode 100644 superpowers/specs/2026-06-22-pyspiral-upgrade-fix-ci-header-signing.md delete mode 100644 superpowers/specs/2026-06-23-plt-1659-extension-type-roundtrip-integration-tests-design.md delete mode 100644 superpowers/specs/2026-06-24-plt-1660-hard-cut-extension-type-hashing.md delete mode 100644 superpowers/specs/2026-06-27-itl-432-pydantic-dataclass-pipeline-columns-design.md delete mode 100644 superpowers/specs/2026-06-28-pick-index-operators-design.md delete mode 100644 superpowers/specs/2026-06-29-itl-45-datetime-tag-values-design.md delete mode 100644 superpowers/specs/2026-06-29-linear-release-system-design.md delete mode 100644 superpowers/specs/2026-06-29-op-file-type-design.md delete mode 100644 superpowers/specs/2026-06-29-typing-literal-support-design.md delete mode 100644 superpowers/specs/2026-06-29-union-typed-function-inputs-design.md delete mode 100644 superpowers/specs/2026-06-30-itl-451-op-directory-type-design.md delete mode 100644 superpowers/specs/2026-06-30-itl-453-union-signature-hash-order-independence-design.md delete mode 100644 superpowers/specs/2026-06-30-itl-460-numpy-ndarray-native-value-type-design.md delete mode 100644 superpowers/specs/2026-06-30-release-workflow-design.md delete mode 100644 superpowers/specs/2026-07-01-pandas-native-types-design.md delete mode 100644 superpowers/specs/2026-07-02-itl-472-file-hash-caching-design.md delete mode 100644 superpowers/specs/2026-07-03-itl-489-warm-cache-design.md delete mode 100644 superpowers/specs/2026-07-03-itl-493-pathlike-conformance-design.md delete mode 100644 superpowers/specs/2026-07-03-itl-concurrent-warm-cache-design.md delete mode 100644 superpowers/specs/2026-07-05-itl-500-populate-hash-cache-ergonomics-design.md delete mode 100644 superpowers/specs/2026-07-07-dict-source-auto-registration-design.md delete mode 100644 superpowers/specs/2026-07-07-itl-474-opfile-url-form-identity-design.md delete mode 100644 superpowers/specs/2026-07-07-itl-507-ephemeral-function-pod-output-design.md delete mode 100644 superpowers/specs/2026-07-08-itl-10-dataclass-dataframe-conversion-tests-design.md delete mode 100644 superpowers/specs/2026-07-08-itl-471-spiraldb-arrow-metadata-preservation-design.md delete mode 100644 superpowers/specs/2026-07-08-itl-508-indexed-entry-id-versioning-design.md delete mode 100644 superpowers/specs/2026-07-08-itl-511-directory-hash-caching-design.md delete mode 100644 superpowers/specs/2026-07-08-itl-513-stale-entry-overwrite-resolution.md delete mode 100644 superpowers/specs/2026-07-08-node-config-pod-config-split-design.md delete mode 100644 superpowers/specs/2026-07-09-itl-473-orcapod-extension-spikeinterface-split-design.md delete mode 100644 superpowers/specs/2026-07-09-itl-516-unify-async-execute-design.md delete mode 100644 superpowers/specs/2026-07-10-itl-519-hash-cacher-write-guards-design.md delete mode 100644 superpowers/specs/2026-07-10-itl-520-postgres-hash-cacher-design.md delete mode 100644 superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md delete mode 100644 superpowers/specs/2026-07-14-itl-523-post-run-hook-design.md delete mode 100644 superpowers/specs/2026-07-14-itl-524-side-effect-pods-design.md delete mode 100644 superpowers/specs/2026-07-14-itl-525-side-effect-pod-impl-design.md delete mode 100644 superpowers/specs/2026-07-16-empty-data-ephemeral-propagation-design.md delete mode 100644 superpowers/specs/2026-07-16-itl-532-side-effect-function-pod-design.md delete mode 100644 superpowers/specs/2026-07-16-itl-533-node-content-hash-preimage-redundancy-design.md delete mode 100644 superpowers/specs/2026-07-17-ctx-arg-and-ray-opts-hardening.md delete mode 100644 superpowers/specs/2026-07-17-itl-535-schema-versioning-migration-design.md delete mode 100644 superpowers/specs/2026-07-21-hash-columns-large-binary-design.md delete mode 100644 superpowers/specs/2026-07-21-itl-529-hash-audit-design.md delete mode 100644 superpowers/specs/2026-07-21-itl-545-release-branch-argument-design.md delete mode 100644 superpowers/specs/2026-07-21-postrunpayload-invocation-context-design.md delete mode 100644 superpowers/specs/2026-07-22-empty-table-nullability-design.md delete mode 100644 superpowers/specs/2026-08-03-itl-602-extension-deserializer-storage-family-design.md delete mode 100644 superpowers/specs/2026-08-05-itl-603-directory-exclude-by-pattern-design.md delete mode 100644 superpowers/specs/2026-08-07-npipe-204-batch-group-by-design.md delete mode 100644 superpowers/specs/2026-08-11-itl-173-list-set-logical-types-design.md delete mode 100644 superpowers/specs/2026-08-20-github-actions-node24-upgrade-design.md delete mode 100644 superpowers/specs/2026-08-20-itl-615-pollingsource-pipelinejob-async-delegation-fix.md delete mode 100644 superpowers/specs/2026-08-22-itl-627-list-extension-metadata.md delete mode 100644 superpowers/specs/2026-08-25-itl-616-polling-source-combine-content-hash-fix.md delete mode 100644 superpowers/specs/2026-08-25-itl-617-pollingsource-async-mode-guard-design.md delete mode 100644 superpowers/specs/2026-08-27-eng-952-polling-source-zero-row-batch-fix.md delete mode 100644 superpowers/specs/2026-09-02-itl-638-stable-type-annotation-hashing-design.md delete mode 100644 superpowers/specs/2026-09-03-itl-639-schema-hashing-arrow-representation-design.md delete mode 100644 superpowers/specs/2026-11-09-itl-817-content-derived-record-ids.md delete mode 100644 superpowers/specs/2026-11-10-array-format-v0.3.md delete mode 100644 superpowers/specs/2026-11-10-user-class-identity-v0.3.md delete mode 100644 superpowers/specs/2026-11-11-itl-186-spiral-streaming-source-design.md delete mode 100644 superpowers/specs/2026-11-11-itl-828-polars-expr-identity-design.md delete mode 100644 superpowers/specs/2026-11-11-itl-897-pandas-hash-identity.md delete mode 100644 superpowers/specs/2026-11-11-itl-938-versioning-system-design.md delete mode 100644 superpowers/specs/2026-11-11-itl-939-tag-and-data-same-name-design.md delete mode 100644 superpowers/specs/2026-11-11-public-release-readiness.md delete mode 100644 superpowers/specs/2026-11-11-torch-tensor-v0.3.md delete mode 100644 superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md delete mode 100644 superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md delete mode 100644 superpowers/specs/2026-11-12-itl-972-user-class-kinds.md delete mode 100644 superpowers/specs/2026-11-12-itl-981-library-value-types-design.md delete mode 100644 superpowers/specs/2026-11-12-itl-991-stream-head-and-preview-design.md delete mode 100644 superpowers/specs/2026-11-13-itl-1074-test-inventory.md delete mode 100644 superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md delete mode 100644 superpowers/specs/2026-11-15-itl-1154-golden-additions.tsv delete mode 100644 superpowers/specs/2026-11-15-itl-1154-golden-moves.md delete mode 100644 superpowers/specs/2026-11-15-itl-1154-golden-moves.tsv delete mode 100644 superpowers/specs/2026-11-15-itl-1154-hash-site-inventory.md delete mode 100644 superpowers/specs/2026-11-15-itl-1154-row-identity-note.md delete mode 100644 superpowers/specs/2026-11-16-itl-1154-key-policy-moves.md delete mode 100644 superpowers/specs/2026-11-16-itl-1154-key-policy-moves.tsv delete mode 100644 superpowers/specs/itl-1034-sweep/README.md delete mode 100644 superpowers/specs/itl-1034-sweep/datagram_uuid_after_readback.py delete mode 100644 superpowers/specs/itl-1034-sweep/interleaved_reads_recompute.py delete mode 100644 superpowers/specs/itl-1034-sweep/itl1043_loaded_operator_cache.py delete mode 100644 superpowers/specs/itl-1034-sweep/itl1045_two_pass_as_table.py delete mode 100644 superpowers/specs/itl-1034-sweep/itl1046_index_token.py delete mode 100644 superpowers/specs/itl-1034-sweep/itl1047_same_stream_twice.py delete mode 100644 superpowers/specs/itl-1034-sweep/partial_read_then_head.py delete mode 100644 superpowers/specs/itl-1034-sweep/side_effect_stream_as_table.py delete mode 100644 superpowers/specs/itl-1040/README.md delete mode 100644 superpowers/specs/itl-1040/measure.py delete mode 100644 superpowers/specs/itl-1042/rowless_head_columns.py delete mode 100644 superpowers/specs/itl-1043/FINDINGS.md delete mode 100644 superpowers/specs/itl-1043/q1_repro.py delete mode 100644 superpowers/specs/itl-1043/q2_where_rows_go.py delete mode 100644 superpowers/specs/itl-1043/q3_schemas.py delete mode 100644 superpowers/specs/itl-1043/q4_blast_radius.py delete mode 100644 superpowers/specs/itl-1043/q4b_from_pipeline.py delete mode 100644 superpowers/specs/itl-1043/q5_nullability_hash.py delete mode 100644 superpowers/specs/itl-1043/q6_live_replay.py delete mode 100644 superpowers/specs/itl-186-experiments/as_table_after_sync_vs_async_run.py delete mode 100644 superpowers/specs/itl-186-experiments/d2_restriction_identity.py delete mode 100644 superpowers/specs/itl-828-experiments/exp1.py delete mode 100644 superpowers/specs/itl-828-experiments/exp2.py delete mode 100644 superpowers/specs/itl-892-exit-hang/FINDINGS.md delete mode 100644 superpowers/specs/itl-892-exit-hang/check_native_fs_equivalence.py delete mode 100644 superpowers/specs/itl-892-exit-hang/harness.py delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round1-py3.11/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round1-py3.12/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round1-py3.13/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round1-py3.13/samples/linux_op=to_table_py3.13_locked.txt delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round2-py3.11/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round2-py3.12/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round2-py3.13/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/linux/round2-py3.13/samples/linux_orcapod_plain-read_patch=none_py3.13_locked.txt delete mode 100644 superpowers/specs/itl-892-exit-hang/native_fs_pytest_plugin.py delete mode 100644 superpowers/specs/itl-892-exit-hang/regression_check_proto.py delete mode 100644 superpowers/specs/itl-892-exit-hang/repro_orcapod.py delete mode 100644 superpowers/specs/itl-892-exit-hang/repro_pure.py delete mode 100644 superpowers/specs/itl-892-exit-hang/results.jsonl delete mode 100644 superpowers/specs/itl-892-exit-hang/run_matrix.sh delete mode 100644 superpowers/specs/itl-892-exit-hang/samples/dataset_scan_py3.13_pa23.0.1_dl1.0.2.txt delete mode 100644 superpowers/specs/itl-892-exit-hang/samples/op=dataset_scan_py3.12_locked.txt delete mode 100644 superpowers/specs/itl-892-exit-hang/samples/orcapod_plain_read_hang_manual.txt delete mode 100644 superpowers/specs/itl-892-exit-hang/summarize.py delete mode 100644 superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py delete mode 100644 superpowers/specs/itl-895-float-to-int/02_call_chain.py delete mode 100644 superpowers/specs/itl-895-float-to-int/03_split_datagram.py delete mode 100644 superpowers/specs/itl-895-float-to-int/04_input_sites.py delete mode 100644 superpowers/specs/itl-895-float-to-int/05_pod_matrix.py delete mode 100644 superpowers/specs/itl-895-float-to-int/06_released_versions.py delete mode 100644 superpowers/specs/itl-895-float-to-int/07_blast_radius.py delete mode 100644 superpowers/specs/itl-895-float-to-int/08_none_output.py delete mode 100644 superpowers/specs/itl-895-float-to-int/09_warning_dedup.py delete mode 100644 superpowers/specs/itl-895-float-to-int/FINDINGS.md delete mode 100644 superpowers/specs/itl-895-float-to-int/_tracer.py delete mode 100644 superpowers/specs/itl-926-live/FINDINGS.md delete mode 100644 superpowers/specs/itl-926-live/error_messages.py delete mode 100644 superpowers/specs/itl-926-live/inspect_tables.py delete mode 100644 superpowers/specs/itl-926-live/live_verify.py delete mode 100644 superpowers/specs/itl-926-live/proposals/c3b_differential.py delete mode 100644 superpowers/specs/itl-926-live/repros/c1_nullable_columns_refuse_plain_pods.py delete mode 100644 superpowers/specs/itl-926-live/repros/c2_fixed_size_list_cannot_hash.py delete mode 100644 superpowers/specs/itl-926-live/repros/c3_source_memory.py delete mode 100644 superpowers/specs/itl-926-live/repros/h1_timestamp_tag_becomes_utc.py delete mode 100644 superpowers/specs/itl-926-live/repros/h2_positional_record_ids.py delete mode 100644 superpowers/specs/itl-926-live/repros/h4_get_all_records_delta_after_run.py delete mode 100644 superpowers/specs/itl-926-live/repros/source_id_collision_across_projects.py delete mode 100644 superpowers/specs/itl-926-live/scan_cost_probe.py delete mode 100644 superpowers/specs/itl-926-live/size_probe.py delete mode 100644 superpowers/specs/itl-936-live/live_nullability.py delete mode 100644 superpowers/specs/itl-939-experiments/exp01_same_name_today.out delete mode 100644 superpowers/specs/itl-939-experiments/exp01_same_name_today.py delete mode 100644 superpowers/specs/itl-939-experiments/exp02_workarounds_today.out delete mode 100644 superpowers/specs/itl-939-experiments/exp02_workarounds_today.py delete mode 100644 superpowers/specs/itl-939-experiments/exp03_copy_tag_prototype.out delete mode 100644 superpowers/specs/itl-939-experiments/exp03_copy_tag_prototype.py delete mode 100644 superpowers/specs/itl-939-experiments/exp04_foreign_key_promotion.out delete mode 100644 superpowers/specs/itl-939-experiments/exp04_foreign_key_promotion.py delete mode 100644 superpowers/specs/itl-939-experiments/exp05_hash_comparison.out delete mode 100644 superpowers/specs/itl-939-experiments/exp05_hash_comparison.py delete mode 100644 superpowers/specs/itl-939-experiments/exp06_derived_source_today.out delete mode 100644 superpowers/specs/itl-939-experiments/exp06_derived_source_today.py delete mode 100644 superpowers/specs/itl-939-experiments/exp07_derived_source_prototype.out delete mode 100644 superpowers/specs/itl-939-experiments/exp07_derived_source_prototype.py delete mode 100644 superpowers/specs/itl-939-experiments/exp08_group_by_tags_into_data.out delete mode 100644 superpowers/specs/itl-939-experiments/exp08_group_by_tags_into_data.py delete mode 100644 superpowers/specs/itl-939-live-check/spiraldb_derived_source_check.py diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index ebffbbc51..bda4c33bd 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -230,6 +230,408 @@ } } ], + "./scripts/hash_audit_example.py": [ + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 38, + "endColumn": 50, + "lineCount": 1 + } + }, + { + "code": "reportPrivateLocalImportUsage", + "range": { + "startColumn": 33, + "endColumn": 50, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 66, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 67, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 70, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 71, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 66, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 67, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 63, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 64, + "lineCount": 1 + } + }, + { + "code": "reportPrivateUsage", + "range": { + "startColumn": 18, + "endColumn": 32, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 15, + "lineCount": 1 + } + }, + { + "code": "reportUnknownArgumentType", + "range": { + "startColumn": 23, + "endColumn": 41, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 17, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 19, + "endColumn": 20, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 15, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 17, + "endColumn": 18, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 13, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 16, + "endColumn": 38, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 11, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 14, + "endColumn": 34, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 63, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 69, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 17, + "lineCount": 1 + } + }, + { + "code": "reportUnknownArgumentType", + "range": { + "startColumn": 25, + "endColumn": 78, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 31, + "endColumn": 32, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 36, + "endColumn": 52, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 36, + "endColumn": 58, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 4, + "endColumn": 17, + "lineCount": 1 + } + }, + { + "code": "reportUnknownArgumentType", + "range": { + "startColumn": 25, + "endColumn": 78, + "lineCount": 1 + } + }, + { + "code": "reportUnknownVariableType", + "range": { + "startColumn": 31, + "endColumn": 32, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 36, + "endColumn": 52, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 36, + "endColumn": 58, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 44, + "endColumn": 77, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 42, + "endColumn": 75, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 42, + "endColumn": 81, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 42, + "endColumn": 73, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 42, + "endColumn": 79, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 21, + "endColumn": 24, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 64, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 47, + "endColumn": 65, + "lineCount": 1 + } + }, + { + "code": "reportUnknownParameterType", + "range": { + "startColumn": 42, + "endColumn": 48, + "lineCount": 1 + } + }, + { + "code": "reportMissingParameterType", + "range": { + "startColumn": 42, + "endColumn": 48, + "lineCount": 1 + } + }, + { + "code": "reportUnusedParameter", + "range": { + "startColumn": 42, + "endColumn": 48, + "lineCount": 1 + } + }, + { + "code": "reportUnusedCallResult", + "range": { + "startColumn": 8, + "endColumn": 38, + "lineCount": 1 + } + }, + { + "code": "reportUnusedCallResult", + "range": { + "startColumn": 4, + "endColumn": 13, + "lineCount": 1 + } + }, + { + "code": "reportPrivateUsage", + "range": { + "startColumn": 51, + "endColumn": 58, + "lineCount": 1 + } + }, + { + "code": "reportPrivateUsage", + "range": { + "startColumn": 48, + "endColumn": 60, + "lineCount": 1 + } + }, + { + "code": "reportUnknownMemberType", + "range": { + "startColumn": 8, + "endColumn": 22, + "lineCount": 1 + } + }, + { + "code": "reportPrivateUsage", + "range": { + "startColumn": 24, + "endColumn": 44, + "lineCount": 1 + } + } + ], "./src/orcapod/__init__.py": [ { "code": "reportImportCycles", @@ -65926,408 +66328,6 @@ } } ], - "./superpowers/scripts/hash_audit_example.py": [ - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 38, - "endColumn": 50, - "lineCount": 1 - } - }, - { - "code": "reportPrivateLocalImportUsage", - "range": { - "startColumn": 33, - "endColumn": 50, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 67, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 70, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 71, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 67, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 64, - "lineCount": 1 - } - }, - { - "code": "reportPrivateUsage", - "range": { - "startColumn": 18, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 15, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 23, - "endColumn": 41, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 19, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 15, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 17, - "endColumn": 18, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 16, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 11, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 14, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 69, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 25, - "endColumn": 78, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 31, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 36, - "endColumn": 52, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 36, - "endColumn": 58, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 25, - "endColumn": 78, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 31, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 36, - "endColumn": 52, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 36, - "endColumn": 58, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 44, - "endColumn": 77, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 42, - "endColumn": 75, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 42, - "endColumn": 81, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 42, - "endColumn": 73, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 42, - "endColumn": 79, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 21, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 64, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 65, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 42, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 42, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportUnusedParameter", - "range": { - "startColumn": 42, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportUnusedCallResult", - "range": { - "startColumn": 8, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnusedCallResult", - "range": { - "startColumn": 4, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportPrivateUsage", - "range": { - "startColumn": 51, - "endColumn": 58, - "lineCount": 1 - } - }, - { - "code": "reportPrivateUsage", - "range": { - "startColumn": 48, - "endColumn": 60, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 22, - "lineCount": 1 - } - }, - { - "code": "reportPrivateUsage", - "range": { - "startColumn": 24, - "endColumn": 44, - "lineCount": 1 - } - } - ], "./test-objective/conftest.py": [ { "code": "reportUnusedParameter", @@ -258841,4 +258841,4 @@ } ] } -} \ No newline at end of file +} diff --git a/.gitignore b/.gitignore index 831c746a4..ea3ee9ef7 100644 --- a/.gitignore +++ b/.gitignore @@ -219,11 +219,5 @@ dj_*_conf.json .pixi/* !.pixi/config.toml -# Superpowers (Claude Code skill artifacts - ignore everything except specs/plans/scripts) -superpowers/* -!superpowers/specs -!superpowers/plans -!superpowers/scripts - # General JSON files at project root /*.json diff --git a/.zed/rules b/.zed/rules index 423a3c2a8..f2095e7e5 100644 --- a/.zed/rules +++ b/.zed/rules @@ -41,14 +41,6 @@ When fixing a bug or addressing a design problem: When discovering a new issue that won't be fixed immediately, ask the user whether it should be logged in design_issues/ before adding it. -## Superpowers artifacts - -Place all superpowers-related artifacts (design specs, plans, etc.) in the superpowers/ -directory at the project root — NOT under docs/. The docs/ directory is reserved for -actual library documentation. - -- Specs go in superpowers/specs/ - ## Backward compatibility This is a greenfield project pre-v0.1.0. Do NOT add backward-compatibility shims, @@ -80,6 +72,8 @@ other ReST cross-reference roles. Use plain ``backtick`` notation to refer to na ## Documentation samples +docs/ holds library documentation only: agent plans, specs and working notes never go there. + Every fenced python block in README.md, docs/index.md, docs/getting-started.md, docs/troubleshooting.md, docs/concepts/, docs/reference/ and docs/tutorial/ is executed by tests/test_docs/test_doc_samples.py, and tests/test_docs/test_mkdocs_build.py runs diff --git a/AGENT_COORDINATION.md b/AGENT_COORDINATION.md index adac28dfa..42b34be1d 100644 --- a/AGENT_COORDINATION.md +++ b/AGENT_COORDINATION.md @@ -109,8 +109,8 @@ developer. If the developer types into the worker's terminal directly, that instruction wins; tell the coordinator with an escalation. - **Plan first.** Follow `CLAUDE.md` and the Sensei skill chain (`work-start`, then - `brainstorming` or `writing-plans`). Save the plan under `superpowers/` before changing - code. For anything beyond a contained bug fix, send the plan summary and open questions + `brainstorming` or `writing-plans`). Put the plan in the Linear issue or the pull + request description, never in a file in the repository, before changing code. For anything beyond a contained bug fix, send the plan summary and open questions through `ask` and wait for the coordinator's approval before implementing. Flag any point that needs the developer's decision explicitly (see "Approving plans" below). - **Tests.** Write the failing regression test first. Run everything through `uv run`. diff --git a/CLAUDE.md b/CLAUDE.md index fcf7fb693..3c241bbaf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,14 +45,6 @@ When fixing a bug or addressing a design problem: When discovering a new issue that won't be fixed immediately, ask the user whether it should be logged in `design_issues/` before adding it. -## Superpowers artifacts - -Place all superpowers-related artifacts (design specs, plans, etc.) in the `superpowers/` -directory at the project root — **not** under `docs/`. The `docs/` directory is reserved for -actual library documentation. - -- Specs go in `superpowers/specs/` - ## Backward compatibility This is a greenfield project pre-v0.1.0. Do **not** add backward-compatibility shims, @@ -84,6 +76,8 @@ to refer to names (e.g. ``PollingSource``, ``iter_data``, ``CursorInvalidatedErr ## Documentation samples +`docs/` holds library documentation only: agent plans, specs and working notes never go there. + Every fenced `python` block in `README.md`, `docs/index.md`, `docs/getting-started.md`, `docs/troubleshooting.md`, `docs/concepts/`, `docs/reference/` and `docs/tutorial/` is executed by `tests/test_docs/test_doc_samples.py`, and `tests/test_docs/test_mkdocs_build.py` runs diff --git a/RELEASING.md b/RELEASING.md index f3e5c72a0..ce61cc856 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -80,8 +80,7 @@ Do these on `metamorphic-labs` `main` before anything is pushed: not exist on PyPI or TestPyPI; neither index allows a version to be uploaded twice, even after it is deleted. -Per-release checklists live in `superpowers/plans/` (for example -`2026-11-09-v0.2-release-checklist.md`). +The per-release checklist lives in the release pull request's description. ## Pre-push checklist diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index aa4f4af78..80b9e7523 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -322,7 +322,7 @@ every content hash is reproducible across runs. Run the script to regenerate the in this section: ``` -uv run python superpowers/scripts/hash_audit_example.py +uv run python scripts/hash_audit_example.py ``` ### Example pipeline diff --git a/superpowers/scripts/hash_audit_example.py b/scripts/hash_audit_example.py similarity index 99% rename from superpowers/scripts/hash_audit_example.py rename to scripts/hash_audit_example.py index 34c0fcfe6..20870ec84 100644 --- a/superpowers/scripts/hash_audit_example.py +++ b/scripts/hash_audit_example.py @@ -6,7 +6,7 @@ all 14 hash site values. Use explicit source_id values so every hash is reproducible across runs. -Run with: uv run python superpowers/scripts/hash_audit_example.py +Run with: uv run python scripts/hash_audit_example.py """ from __future__ import annotations diff --git a/src/orcapod/pipeline/dag.py b/src/orcapod/pipeline/dag.py index 8f550d432..81c8ffee5 100644 --- a/src/orcapod/pipeline/dag.py +++ b/src/orcapod/pipeline/dag.py @@ -4,9 +4,6 @@ covering exactly the nine API shapes Orcapod requires. Backed by plain dicts and `graphlib.TopologicalSorter` from the Python standard library. -See superpowers/specs/2026-05-21-networkx-replacement-design.md for the full -rationale and migration map. - Example: >>> dag: OrcaDAG[str] = OrcaDAG() >>> dag.add_edge("a", "b") diff --git a/superpowers/plans/2026-05-12-data-to-data-rename.md b/superpowers/plans/2026-05-12-data-to-data-rename.md deleted file mode 100644 index ef5535260..000000000 --- a/superpowers/plans/2026-05-12-data-to-data-rename.md +++ /dev/null @@ -1,614 +0,0 @@ -# Rename `packets` → `data` Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Rename every `packet`/`Packet`/`PACKET_*` identifier to `data`/`Data`/`DATA_*` across orcapod-python and land it as one atomic commit. - -**Architecture:** A Python script applies an ordered substitution table to all text files in the repo; then `git mv` renames the affected files and directory. The existing test suite verifies the rename. A single atomic commit captures all changes including a new `CHANGELOG.md`. - -**Tech Stack:** Python 3.x (rename script), Git (file renames + commit), pytest via `uv run` - -**Spec:** `superpowers/specs/2026-05-12-packet-to-data-rename-design.md` - ---- - -### Task 1: Check out the feature branch - -**Files:** (git operation only) - -- [ ] **Step 1: Check out the feature branch** - -```bash -git checkout -b eywalker/eng-454-rename-packets-data-across-orcapod-python-hard-break -``` - -Expected: -``` -Switched to a new branch 'eywalker/eng-454-rename-packets-data-across-orcapod-python-hard-break' -``` - -- [ ] **Step 2: Verify you are on the correct branch** - -```bash -git branch --show-current -``` - -Expected: -``` -eywalker/eng-454-rename-packets-data-across-orcapod-python-hard-break -``` - ---- - -### Task 2: Create `scripts/rename_packet_to_data.py` - -**Files:** -- Create: `scripts/rename_packet_to_data.py` - -This script is a one-shot tool that rewrites file contents. File/directory renames are done separately via `git mv` in Task 4. Delete the script before committing (Task 8). - -- [ ] **Step 1: Create the `scripts/` directory and write the script** - -Create `scripts/rename_packet_to_data.py` with the following content: - -```python -#!/usr/bin/env python3 -"""One-shot rename script: packet → data across orcapod-python. - -Run from the repo root: - python scripts/rename_packet_to_data.py - -After running, use git mv to rename files (Task 4 of the implementation plan). -Delete this script before committing. -""" - -from pathlib import Path - -# ── Substitutions in longest/most-specific first order ────────────────────── -# Plain str.replace — no regex. Order is significant: longer patterns must -# precede shorter ones that are substrings of them (e.g. `packet_function` -# must come before `packet`, `output_packet_schema_hash` before -# `output_packet_schema`). -SUBSTITUTIONS: list[tuple[str, str]] = [ - # SCREAMING_SNAKE_CASE system constants - # (all-caps PACKET is not matched by the PascalCase or lowercase rules below) - ("INPUT_PACKET_HASH_COL", "INPUT_DATA_HASH_COL"), - ("PACKET_RECORD_ID", "DATA_RECORD_ID"), - - # PascalCase class names — longest first - ("PacketFunctionExecutorProtocol", "DataFunctionExecutorProtocol"), - ("PacketExecutionLoggerProtocol", "DataExecutionLoggerProtocol"), - ("PacketFunctionWrapper", "DataFunctionWrapper"), - ("CachedPacketFunction", "CachedDataFunction"), - ("PythonPacketFunction", "PythonDataFunction"), - ("PacketFunctionProtocol", "DataFunctionProtocol"), - ("PacketFunctionProxy", "DataFunctionProxy"), - ("PacketFunctionBase", "DataFunctionBase"), - ("PacketFunction", "DataFunction"), - ("SelectPacketColumns", "SelectDataColumns"), - ("DropPacketColumns", "DropDataColumns"), - ("MapPackets", "MapData"), - ("PacketLogger", "DataLogger"), - ("PacketProtocol", "DataProtocol"), - ("Packet", "Data"), - - # snake_case identifiers — longest first - ("packet_function_type_id", "data_function_type_id"), - ("input_packet_schema", "input_data_schema"), - ("output_packet_schema_hash", "output_data_schema_hash"), - ("output_packet_schema", "output_data_schema"), - ("_async_process_packet_internal", "_async_process_data_internal"), - ("_process_packet_internal", "_process_data_internal"), - ("_iter_packets_sequential", "_iter_data_sequential"), - ("_iter_packets_concurrent", "_iter_data_concurrent"), - ("_async_execute_one_packet", "_async_execute_one_data"), - ("_build_output_packet", "_build_output_data"), - ("get_cached_output_for_packet", "get_cached_output_for_data"), - ("record_packet", "record_data"), - ("async_process_packet", "async_process_data"), - ("process_packet", "process_data"), - ("execute_packet", "execute_data"), - ("async_iter_packets", "async_iter_data"), - ("iter_packets", "iter_data"), - ("map_packets", "map_data"), - ("select_packet_columns", "select_data_columns"), - ("drop_packet_columns", "drop_data_columns"), - ("packet_function", "data_function"), # after packet_function_type_id - ("on_packet_start", "on_data_start"), - ("on_packet_end", "on_data_end"), - ("on_packet_crash", "on_data_crash"), - ("create_packet_logger", "create_data_logger"), - ("verify_packet_schema", "verify_data_schema"), - ("_build_packet_function_registry", "_build_data_function_registry"), - ("resolve_packet_function_from_config", "resolve_data_function_from_config"), - ("register_packet_function", "register_data_function"), - ("packet_schema", "data_schema"), - ("input_packet", "input_data"), - ("output_packet", "output_data"), - ("packets", "data"), # plural before singular - ("packet", "data"), # catch-all — must be last -] - -# File extensions to process -EXTENSIONS = frozenset({ - ".py", ".md", ".rst", ".ipynb", ".toml", - ".yaml", ".yml", ".ini", ".cfg", ".txt", -}) - -# Known extensionless files to also process (relative to repo root) -EXTRA_FILES = [ - ".zed/rules", -] - -# Directories whose entire subtree is skipped -SKIP_DIRS = frozenset({ - ".git", ".venv", "venv", "__pycache__", ".mypy_cache", - "node_modules", ".tox", "dist", "build", - "superpowers", # design docs — preserve old names as historical reference -}) - - -def apply_substitutions(content: str) -> str: - for old, new in SUBSTITUTIONS: - content = content.replace(old, new) - return content - - -def process_file(path: Path) -> bool: - """Apply substitutions to one file. Returns True if the file changed.""" - try: - original = path.read_text(encoding="utf-8") - except (UnicodeDecodeError, PermissionError) as exc: - print(f" SKIP {path}: {exc}") - return False - updated = apply_substitutions(original) - if updated != original: - path.write_text(updated, encoding="utf-8") - return True - return False - - -def main() -> None: - repo_root = Path(__file__).resolve().parent.parent - changed: list[Path] = [] - - # Walk all files with matching extensions - for path in sorted(repo_root.rglob("*")): - if any(part in SKIP_DIRS for part in path.parts): - continue - if path.resolve() == Path(__file__).resolve(): - continue - if path.is_file() and path.suffix in EXTENSIONS: - if process_file(path): - changed.append(path.relative_to(repo_root)) - - # Process known extensionless files - for rel in EXTRA_FILES: - path = repo_root / rel - if path.exists() and path.is_file(): - if process_file(path): - changed.append(path.relative_to(repo_root)) - - print(f"\nModified {len(changed)} file(s):") - for p in sorted(changed): - print(f" {p}") - - -if __name__ == "__main__": - main() -``` - ---- - -### Task 3: Run the rename script (content substitution) - -**Files:** ~186 files modified in-place by the script - -- [ ] **Step 1: Run the script from the repo root** - -```bash -python scripts/rename_packet_to_data.py -``` - -Expected output (abbreviated — exact count may vary): -``` -Modified 186 file(s): - CLAUDE.md - DESIGN_ISSUES.md - README.md - docs/... - examples/... - notebooks/... - src/orcapod/core/datagrams/tag_packet.py - src/orcapod/core/data_function.py ← content already updated, name still old - ... - tests/test_core/packet_function/test_cached_packet_function.py - ... -``` - -Note: file names in the output still show the old names — that is correct at this stage. -File renames happen via `git mv` in Task 4. - -- [ ] **Step 2: Spot-check a few critical files to confirm substitutions applied** - -```bash -grep -n "DataFunctionBase\|DataProtocol\|iter_data\|data_function" \ - src/orcapod/core/datagrams/tag_packet.py \ - src/orcapod/core/packet_function.py \ - src/orcapod/protocols/core_protocols/packet_function.py | head -30 -``` - -Expected: lines containing the NEW names (`DataFunctionBase`, `iter_data`, etc.), not the old ones. - -- [ ] **Step 3: Confirm no bare `packet` / `Packet` remain in source or docs** - -```bash -# Python source and tests -grep -rn --include="*.py" "packet\|Packet" src/ tests/ | head -20 - -# Markdown docs and examples -grep -rn --include="*.md" "packet\|Packet" docs/ examples/ notebooks/ README.md \ - DESIGN_ISSUES.md CLAUDE.md 2>/dev/null | head -20 -``` - -Expected: zero matches in both commands. If any remain, check whether they are genuine -misses or false positives (e.g. a comment that uses "packet" in a non-identifier context). -Fix genuine misses by adding them to `SUBSTITUTIONS` and re-running the script. - ---- - -### Task 4: Find all files with `packet` in their path name, then rename them - -**Files:** (git mv operations) - -- [ ] **Step 1: List all git-tracked files with `packet` in their path** - -```bash -git ls-files | grep -i packet -``` - -Expected output (these are the files that need `git mv`): -``` -src/orcapod/core/datagrams/tag_packet.py -src/orcapod/core/packet_function.py -src/orcapod/core/packet_function_proxy.py -src/orcapod/protocols/core_protocols/packet_function.py -tests/test_core/packet_function/__init__.py -tests/test_core/packet_function/test_cached_packet_function.py -tests/test_core/packet_function/test_executor.py -tests/test_core/packet_function/test_packet_function.py -tests/test_core/packet_function/test_packet_function_config.py -tests/test_core/packet_function/test_packet_function_proxy.py -``` - -If there are more files than expected, rename them with the same `packet` → `data` pattern -before proceeding. - -- [ ] **Step 2: Rename individual test files first (while still in the old directory)** - -```bash -git mv tests/test_core/packet_function/test_cached_packet_function.py \ - tests/test_core/packet_function/test_cached_data_function.py - -git mv tests/test_core/packet_function/test_packet_function.py \ - tests/test_core/packet_function/test_data_function.py - -git mv tests/test_core/packet_function/test_packet_function_config.py \ - tests/test_core/packet_function/test_data_function_config.py - -git mv tests/test_core/packet_function/test_packet_function_proxy.py \ - tests/test_core/packet_function/test_data_function_proxy.py -``` - -(`test_executor.py` has no `packet` in its name — no rename needed.) - -- [ ] **Step 3: Rename the test directory** - -```bash -git mv tests/test_core/packet_function tests/test_core/data_function -``` - -- [ ] **Step 4: Rename the source files** - -```bash -git mv src/orcapod/core/datagrams/tag_packet.py \ - src/orcapod/core/datagrams/tag_data.py - -git mv src/orcapod/core/packet_function.py \ - src/orcapod/core/data_function.py - -git mv src/orcapod/core/packet_function_proxy.py \ - src/orcapod/core/data_function_proxy.py - -git mv src/orcapod/protocols/core_protocols/packet_function.py \ - src/orcapod/protocols/core_protocols/data_function.py -``` - -- [ ] **Step 5: Verify no `packet` remains in any tracked path name** - -```bash -git ls-files | grep -i packet -``` - -Expected: no output (zero matches). - -- [ ] **Step 6: Confirm the renamed paths exist and are staged** - -```bash -git status --short | grep -E "^R|^A|^M" | grep -E "data_function|tag_data" | head -20 -``` - -Expected: each renamed file appears as `R old/path -> new/path`. - ---- - -### Task 5: Review and fix same-scope `data` collisions - -**Files:** Any source files where the rename introduces a same-scope name clash - -- [ ] **Step 1: Find any function/method bodies that now have two `data` identifiers in the same scope** - -```bash -grep -n "\bdata\b" src/orcapod/core/data_function.py \ - src/orcapod/core/cached_function_pod.py \ - src/orcapod/core/function_pod.py \ - src/orcapod/pipeline/observer.py \ - src/orcapod/pipeline/logging_observer.py | \ - grep -v "data_function\|data_schema\|input_data\|output_data\|data_context\|iter_data\|process_data\|data_logger\|DataFunction\|DataProtocol\|DataLogger\|variation_data\|exec_data" | head -30 -``` - -For any true same-scope clash (e.g. a local variable named `data` in a function that also -takes a `data` parameter), rename the pre-existing variable to something more specific. - -Example — if you find a method like: -```python -def process_data(self, data: DataProtocol) -> dict: - data = self._fetch_variation_data() # clash: shadowing parameter - return data -``` - -Fix it to: -```python -def process_data(self, data: DataProtocol) -> dict: - variation_data = self._fetch_variation_data() - return variation_data -``` - -- [ ] **Step 2: Confirm no regressions from collision fixes by running a quick import check** - -```bash -uv run python -c "import orcapod; print('import OK')" -``` - -Expected: -``` -import OK -``` - -If this raises an `ImportError` or `ModuleNotFoundError`, the most likely cause is a -`__init__.py` still importing from the old module path (e.g. `from .packet_function import -...`). Fix by checking which `__init__.py` still references the old name: - -```bash -grep -rn "packet_function\|tag_packet" src/orcapod/ -``` - -Any matches are `__init__.py` lines that the rename script missed. Update them manually. - ---- - -### Task 6: Run the full test suite - -**Files:** (read-only — verify only) - -- [ ] **Step 1: Run all tests** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: -``` -... passed, ... warnings in ...s -``` - -- [ ] **Step 2: If any tests fail, diagnose and fix** - -The most common failure modes after a rename: - -a) **ImportError** — a file still imports from the old name. Fix: - ```bash - grep -rn "from.*packet\|import.*packet" src/ tests/ - ``` - Update each remaining reference manually. - -b) **AttributeError on `iter_packets` / `process_packet`** — a test or source file uses - the old method name. Fix by searching and replacing the specific occurrence. - -c) **FileNotFoundError on a module path** — a string-based module reference (e.g. in - `serialization.py`) still uses the old name. Fix by searching for string literals: - ```bash - grep -rn '"packet_function"\|"tag_packet"' src/ tests/ - ``` - -Repeat `uv run pytest tests/ -x -q` after each fix until all tests pass. - -- [ ] **Step 3: Run the full suite without `-x` to see the complete picture** - -```bash -uv run pytest tests/ -q 2>&1 | tail -5 -``` - -Expected: `N passed` with no failures or errors. - ---- - -### Task 7: Create `CHANGELOG.md` - -**Files:** -- Create: `CHANGELOG.md` (repo root) - -- [ ] **Step 1: Create `CHANGELOG.md` at the repo root** - -```markdown -# Changelog - -## [Unreleased] - -### Breaking Changes - -#### `packets` → `data` rename (hard break) - -All identifiers containing `packet`/`packets`/`Packet` have been renamed to -`data`/`Data`. No deprecation aliases. Pre-v0.1 artifacts will not load. - -| Old name | New name | -|---|---| -| `Packet` | `Data` | -| `PacketProtocol` | `DataProtocol` | -| `PacketFunction` | `DataFunction` | -| `PacketFunctionBase` | `DataFunctionBase` | -| `PacketFunctionProtocol` | `DataFunctionProtocol` | -| `PacketFunctionProxy` | `DataFunctionProxy` | -| `PythonPacketFunction` | `PythonDataFunction` | -| `CachedPacketFunction` | `CachedDataFunction` | -| `PacketFunctionWrapper` | `DataFunctionWrapper` | -| `PacketFunctionExecutorProtocol` | `DataFunctionExecutorProtocol` | -| `PacketExecutionLoggerProtocol` | `DataExecutionLoggerProtocol` | -| `PacketLogger` | `DataLogger` | -| `SelectPacketColumns` | `SelectDataColumns` | -| `DropPacketColumns` | `DropDataColumns` | -| `MapPackets` | `MapData` | -| `iter_packets()` | `iter_data()` | -| `process_packet()` | `process_data()` | -| `async_process_packet()` | `async_process_data()` | -| `execute_packet()` | `execute_data()` | -| `map_packets()` | `map_data()` | -| `select_packet_columns()` | `select_data_columns()` | -| `drop_packet_columns()` | `drop_data_columns()` | -| `on_packet_start()` | `on_data_start()` | -| `on_packet_end()` | `on_data_end()` | -| `on_packet_crash()` | `on_data_crash()` | -| `INPUT_PACKET_HASH_COL` | `INPUT_DATA_HASH_COL` | -| `PACKET_RECORD_ID` | `DATA_RECORD_ID` | -``` - ---- - -### Task 8: Delete the rename script and create the atomic commit - -**Files:** -- Delete: `scripts/rename_packet_to_data.py` -- Commit: all staged changes - -- [ ] **Step 1: Confirm tests still pass (final check before committing)** - -```bash -uv run pytest tests/ -q 2>&1 | tail -5 -``` - -Expected: all passing, zero failures. - -- [ ] **Step 2: Delete the rename script** - -```bash -rm scripts/rename_packet_to_data.py -``` - -Remove the `scripts/` directory too if it is now empty: - -```bash -rmdir scripts/ 2>/dev/null || true -``` - -- [ ] **Step 3: Stage all changes** - -```bash -git add -A -``` - -- [ ] **Step 4: Verify the staged file count looks right** - -```bash -git diff --cached --stat | tail -5 -``` - -Expected: ~190+ files changed (186+ content edits + 9 file renames + `CHANGELOG.md`). -The rename script itself should NOT appear (it was deleted and never committed). - -- [ ] **Step 5: Create the single atomic commit** - -```bash -git commit -m "$(cat <<'EOF' -refactor: rename packet → data across orcapod-python (ENG-454) - -Hard break — no deprecation aliases, no migration shims. -All public and private identifiers, docstrings, comments, -module names, and file names containing packet/Packet renamed -to data/Data. Adds CHANGELOG.md with full mapping table. - -Fixes ENG-454 -EOF -)" -``` - -- [ ] **Step 6: Verify the commit looks correct** - -```bash -git show --stat HEAD | head -20 -``` - -Expected: the commit message shows `refactor: rename packet → data...` and the stat lists -the renamed files (shown as `old_name => new_name`). - -- [ ] **Step 7: Push the branch** - -```bash -git push -u origin eywalker/eng-454-rename-packets-data-across-orcapod-python-hard-break -``` - ---- - -### Task 9: Open the pull request - -- [ ] **Step 1: Create the PR targeting `dev`** - -```bash -gh pr create \ - --base dev \ - --title "refactor: rename packet → data across orcapod-python" \ - --body "$(cat <<'EOF' -## Summary - -Hard break rename of all `packet`/`Packet` identifiers to `data`/`Data` throughout -orcapod-python, landing before v0.1 stable. - -- All public + private symbols, module names, file names, docstrings, and comments updated -- No deprecation shims, no backwards-compat aliases -- `CHANGELOG.md` created with full name-mapping table -- Test suite passing - -Fixes ENG-454 - -## Files renamed - -| Old | New | -|---|---| -| `src/orcapod/core/datagrams/tag_packet.py` | `tag_data.py` | -| `src/orcapod/core/packet_function.py` | `data_function.py` | -| `src/orcapod/core/packet_function_proxy.py` | `data_function_proxy.py` | -| `src/orcapod/protocols/core_protocols/packet_function.py` | `data_function.py` | -| `tests/test_core/packet_function/` | `tests/test_core/data_function/` | - -## Test plan -- [ ] `uv run pytest tests/ -q` passes with zero failures -- [ ] `grep -rn "packet\|Packet" src/ tests/` returns zero matches -- [ ] `git ls-files | grep -i packet` returns zero matches -EOF -)" -``` - -- [ ] **Step 2: Record the PR URL** - -Copy the URL printed by `gh pr create` and post it to the Linear issue comment or PR -description so reviewers can find it. diff --git a/superpowers/plans/2026-05-20-pipeline-job-source-spec.md b/superpowers/plans/2026-05-20-pipeline-job-source-spec.md deleted file mode 100644 index a0df86b85..000000000 --- a/superpowers/plans/2026-05-20-pipeline-job-source-spec.md +++ /dev/null @@ -1,2742 +0,0 @@ -# Pipeline + SourceSpec Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Refactor `Pipeline` into a pure source-agnostic DAG; introduce `SourceSpec` (typed input slot) and `PipelineJob` (everyday working object with source bindings, databases, and execution). - -**Architecture:** `SourceSpec` is a named, immutable schema declaration implementing a minimal `StreamProtocol` surface. `Pipeline` is the pure DAG (SourceSpec-only leaves, no databases, no execution). `PipelineJob` extends `AutoRegisteringContextBasedTracker` — its `with`-block records both DAG structure and source bindings simultaneously; concrete sources are automatically promoted to SourceSpecs. `Pipeline` is derived from any `PipelineJob` via `job.pipeline`. All mutation needed for execution happens in fresh node objects inside `PipelineJob._build_execution_graph()`, so the shared `Pipeline` is never mutated. - -**Tech Stack:** Python, PyArrow, NetworkX, `uv run pytest` for tests; follows Google-style docstrings and Conventional Commits. - ---- - -## File Structure - -**New files:** -- `src/orcapod/core/sources/source_spec.py` — `SourceSpec` class -- `src/orcapod/pipeline/execution_context.py` — `ExecutionContext` stub dataclass -- `src/orcapod/pipeline/job.py` — `PipelineJob` class -- `tests/test_core/sources/test_source_spec.py` — unit tests for `SourceSpec` -- `tests/test_pipeline/test_pipeline_job.py` — unit and integration tests for `PipelineJob` - -**Modified files:** -- `src/orcapod/errors.py` — add `UnboundSourceError`, `SourceSpecMismatchError` -- `src/orcapod/core/sources/__init__.py` — export `SourceSpec` -- `src/orcapod/pipeline/graph.py` — refactor `Pipeline` (remove DB/execution, enforce SourceSpec, add `bind()`) -- `src/orcapod/pipeline/serialization.py` — add `PIPELINE_JOB_FORMAT_VERSION`, SourceSpec serialization helpers -- `src/orcapod/pipeline/__init__.py` — export `PipelineJob`, `SourceSpec`, `ExecutionContext` -- `src/orcapod/__init__.py` — export `PipelineJob`, `SourceSpec` -- `tests/test_pipeline/test_pipeline.py` — update all fixtures/tests to new Pipeline (no DB), migrate execution tests to PipelineJob -- `tests/test_pipeline/test_serialization.py` — update to new Pipeline save format; move DB round-trip tests to `test_pipeline_job.py` - ---- - -## Task 1: Error Types - -**Files:** -- Modify: `src/orcapod/errors.py` - -- [ ] **Step 1: Add the two new error classes** - -```python -# Append to src/orcapod/errors.py - -class UnboundSourceError(RuntimeError): - """Raised when a data-producing method is called on an unbound SourceSpec. - - Occurs when ``iter_data()`` or ``as_table()`` is called on a ``SourceSpec`` - that has not been bound to a concrete source in a ``PipelineJob``. - """ - - -class SourceSpecMismatchError(ValueError): - """Raised when a concrete source's schema is incompatible with a SourceSpec. - - Contains the spec name and a description of the incompatible field(s). - Raised at ``bind()`` time — schema mismatches are rejected before execution. - """ -``` - -- [ ] **Step 2: Verify import works** - -Run: `uv run python -c "from orcapod.errors import UnboundSourceError, SourceSpecMismatchError; print('OK')` -Expected: `OK` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/errors.py -git commit -m "feat(errors): add UnboundSourceError and SourceSpecMismatchError" -``` - ---- - -## Task 2: `SourceSpec` Class - -**Files:** -- Create: `src/orcapod/core/sources/source_spec.py` -- Create: `tests/test_core/sources/test_source_spec.py` - -- [ ] **Step 1: Write the failing tests** - -```python -# tests/test_core/sources/test_source_spec.py -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.sources import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec -from orcapod.errors import UnboundSourceError, SourceSpecMismatchError -from orcapod.types import Schema - - -def _make_source(tag_col: str, data_col: str) -> ArrowTableSource: - table = pa.table( - { - tag_col: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array([1, 2], type=pa.int64()), - } - ) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) - - -class TestSourceSpecConstruction: - def test_construct_with_name_and_schemas(self): - spec = SourceSpec( - name="my_source", - tag_schema=Schema({"key": str}), - data_schema=Schema({"value": int}), - ) - assert spec.name == "my_source" - assert "key" in spec.tag_schema - assert "value" in spec.data_schema - - def test_output_schema_returns_tag_and_data(self): - tag = Schema({"key": str}) - data = Schema({"value": int}) - spec = SourceSpec(name="s", tag_schema=tag, data_schema=data) - out_tag, out_data = spec.output_schema() - assert out_tag == tag - assert out_data == data - - def test_keys_returns_tag_column_names(self): - spec = SourceSpec( - name="s", - tag_schema=Schema({"key": str, "group": str}), - data_schema=Schema({"value": int}), - ) - assert spec.keys() == frozenset({"key", "group"}) - - def test_label_returns_name(self): - spec = SourceSpec(name="my_spec", tag_schema=Schema({"k": str}), data_schema=Schema({"v": int})) - assert spec.label == "my_spec" - - -class TestSourceSpecHashing: - def test_pipeline_hash_matches_compatible_source(self): - """SourceSpec.pipeline_hash() must equal a schema-compatible source's pipeline_hash().""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name="my_source", tag_schema=tag_schema, data_schema=data_schema) - - assert spec.pipeline_hash() == source.pipeline_hash() - - def test_content_hash_differs_by_name(self): - """Two specs with the same schema but different names must have different content hashes.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec_a = SourceSpec(name="source_a", tag_schema=tag_schema, data_schema=data_schema) - spec_b = SourceSpec(name="source_b", tag_schema=tag_schema, data_schema=data_schema) - - assert spec_a.content_hash() != spec_b.content_hash() - - def test_pipeline_hash_same_for_same_name_diff_name(self): - """SourceSpec.pipeline_hash() must be schema-only (ignoring name).""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec_a = SourceSpec(name="a", tag_schema=tag_schema, data_schema=data_schema) - spec_b = SourceSpec(name="b", tag_schema=tag_schema, data_schema=data_schema) - - assert spec_a.pipeline_hash() == spec_b.pipeline_hash() - - def test_content_hash_stable(self): - """Same name + schemas → same content hash across calls.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name="s", tag_schema=tag_schema, data_schema=data_schema) - assert spec.content_hash() == spec.content_hash() - - -class TestSourceSpecUnboundBehavior: - def test_iter_data_raises_unbound_error(self): - spec = SourceSpec(name="s", tag_schema=Schema({"k": str}), data_schema=Schema({"v": int})) - with pytest.raises(UnboundSourceError, match="s"): - list(spec.iter_data()) - - def test_as_table_raises_unbound_error(self): - spec = SourceSpec(name="s", tag_schema=Schema({"k": str}), data_schema=Schema({"v": int})) - with pytest.raises(UnboundSourceError, match="s"): - spec.as_table() - - -class TestSourceSpecValidate: - def test_validate_passes_for_compatible_source(self): - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name="s", tag_schema=tag_schema, data_schema=data_schema) - # Should not raise - spec.validate(source) - - def test_validate_raises_for_extra_data_column(self): - """Source has extra data column not in spec → SourceSpecMismatchError.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - # Spec declares only a subset of columns - narrow_data = Schema({"value": int}) - extra_tag = Schema({"key": str, "unexpected": str}) - spec = SourceSpec(name="s", tag_schema=extra_tag, data_schema=narrow_data) - with pytest.raises(SourceSpecMismatchError): - spec.validate(source) - - def test_validate_raises_for_missing_data_column(self): - """Source missing a required data column → SourceSpecMismatchError.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - # Spec requires an extra data column the source doesn't have - wider_data = Schema({"value": int, "extra": str}) - spec = SourceSpec(name="s", tag_schema=tag_schema, data_schema=wider_data) - with pytest.raises(SourceSpecMismatchError): - spec.validate(source) -``` - -- [ ] **Step 2: Run to verify failures** - -Run: `uv run pytest tests/test_core/sources/test_source_spec.py -v` -Expected: ERRORS (module not found) - -- [ ] **Step 3: Implement `SourceSpec`** - -```python -# src/orcapod/core/sources/source_spec.py -"""SourceSpec — a named, immutable schema declaration for pipeline input slots.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -import orcapod.contexts as contexts -from orcapod.core.base import ContentIdentifiableBase, PipelineElementBase -from orcapod.errors import SourceSpecMismatchError, UnboundSourceError -from orcapod.types import ColumnConfig, Schema - -if TYPE_CHECKING: - import pyarrow as pa - - from orcapod.protocols.core_protocols import StreamProtocol - from orcapod.types import ContentHash - - -class SourceSpec(ContentIdentifiableBase, PipelineElementBase): - """A named, immutable schema declaration for a pipeline input slot. - - ``SourceSpec`` describes what a pipeline input looks like — its key schema - and data schema — without referencing any concrete data source. It is used - as the typed input slot concept for both ``Pipeline`` and ``PipelineJob``. - - A ``SourceSpec`` can appear as an upstream in operator chains during a - ``with Pipeline:`` or ``with PipelineJob:`` recording block. Calling data- - producing methods (``iter_data``, ``as_table``) raises ``UnboundSourceError`` - until the spec is bound to a concrete source via ``PipelineJob.bind()``. - - Identity and hashing: - - ``pipeline_hash()`` — schema-only, ignoring ``name``. Matches a - schema-compatible ``RootSource.pipeline_hash()``, enabling DB path - reuse across different sources bound to the same spec. - - ``content_hash()`` — includes ``name``. Two specs with identical - schemas but different names are distinct elements. - - Args: - name: Human-readable identifier for this input slot. Used as the - source label when auto-promoting concrete sources in - ``PipelineJob``. Must be unique within a pipeline. - tag_schema: Mapping of tag column names to Python types. - data_schema: Mapping of data column names to Python types. - """ - - def __init__( - self, - name: str, - tag_schema: Schema, - data_schema: Schema, - data_context: str | contexts.DataContext | None = None, - ) -> None: - super().__init__(data_context=data_context) - self._name = name - self._tag_schema = tag_schema - self._data_schema = data_schema - - # ------------------------------------------------------------------ - # Identity - # ------------------------------------------------------------------ - - @property - def name(self) -> str: - """Human-readable name for this input slot.""" - return self._name - - @property - def label(self) -> str: - """Alias for ``name`` — satisfies the stream label convention.""" - return self._name - - @property - def tag_schema(self) -> Schema: - """Key schema for this input slot.""" - return self._tag_schema - - @property - def data_schema(self) -> Schema: - """Data schema for this input slot.""" - return self._data_schema - - # ------------------------------------------------------------------ - # ContentIdentifiableBase - # ------------------------------------------------------------------ - - def identity_structure(self) -> Any: - """Content identity includes name + both schemas.""" - return ("SourceSpec", self._name, self._tag_schema, self._data_schema) - - # ------------------------------------------------------------------ - # PipelineElementBase - # ------------------------------------------------------------------ - - def pipeline_identity_structure(self) -> Any: - """Pipeline identity is schema-only (no name). - - Matches ``RootSource.pipeline_identity_structure()`` so that a - schema-compatible concrete source and this spec share the same - pipeline hash — and therefore the same DB table paths. - """ - return (self._tag_schema, self._data_schema) - - # ------------------------------------------------------------------ - # StreamProtocol surface (minimal — no data access) - # ------------------------------------------------------------------ - - def output_schema(self, columns: ColumnConfig | None = None) -> tuple[Schema, Schema]: - """Return ``(tag_schema, data_schema)``. - - Args: - columns: Ignored — SourceSpec always returns the full declared schemas. - - Returns: - Tuple of ``(tag_schema, data_schema)``. - """ - return (self._tag_schema, self._data_schema) - - def keys(self, columns: ColumnConfig | None = None) -> frozenset[str]: - """Return the set of tag column names. - - Args: - columns: Ignored. - - Returns: - Frozenset of tag field names. - """ - return frozenset(self._tag_schema.keys()) - - def iter_data(self, *args: Any, **kwargs: Any): - """Raise ``UnboundSourceError`` — spec is not bound to a concrete source. - - Raises: - UnboundSourceError: Always. - """ - raise UnboundSourceError( - f"SourceSpec '{self._name}' is not bound to a concrete source. " - "Call PipelineJob.bind(sources={...}) to attach a source before running." - ) - - def as_table(self, *args: Any, **kwargs: Any) -> "pa.Table": - """Raise ``UnboundSourceError`` — spec is not bound to a concrete source. - - Raises: - UnboundSourceError: Always. - """ - raise UnboundSourceError( - f"SourceSpec '{self._name}' is not bound to a concrete source. " - "Call PipelineJob.bind(sources={...}) to attach a source before running." - ) - - # ------------------------------------------------------------------ - # Validation - # ------------------------------------------------------------------ - - def validate(self, source: "StreamProtocol") -> None: - """Check that *source* is schema-compatible with this spec. - - Validates that the source's tag and data schemas have exactly the - same columns as this spec (no extra, no missing columns). Type - compatibility is not checked here — Arrow conversion handles - coercions at runtime. - - Args: - source: The concrete source to validate. - - Raises: - SourceSpecMismatchError: If the source schema does not match. - """ - source_tag, source_data = source.output_schema() - - tag_issues: list[str] = [] - data_issues: list[str] = [] - - spec_tag_cols = set(self._tag_schema.keys()) - src_tag_cols = set(source_tag.keys()) - if spec_tag_cols != src_tag_cols: - missing = spec_tag_cols - src_tag_cols - extra = src_tag_cols - spec_tag_cols - if missing: - tag_issues.append(f"missing tag columns: {sorted(missing)}") - if extra: - tag_issues.append(f"unexpected tag columns: {sorted(extra)}") - - spec_data_cols = set(self._data_schema.keys()) - src_data_cols = set(source_data.keys()) - if spec_data_cols != src_data_cols: - missing = spec_data_cols - src_data_cols - extra = src_data_cols - spec_data_cols - if missing: - data_issues.append(f"missing data columns: {sorted(missing)}") - if extra: - data_issues.append(f"unexpected data columns: {sorted(extra)}") - - if tag_issues or data_issues: - all_issues = tag_issues + data_issues - raise SourceSpecMismatchError( - f"SourceSpec '{self._name}' is not compatible with the provided source. " - + "; ".join(all_issues) - ) - - # ------------------------------------------------------------------ - # Repr - # ------------------------------------------------------------------ - - def __repr__(self) -> str: - return ( - f"SourceSpec(name={self._name!r}, " - f"tag_schema={dict(self._tag_schema)!r}, " - f"data_schema={dict(self._data_schema)!r})" - ) -``` - -- [ ] **Step 4: Run tests** - -Run: `uv run pytest tests/test_core/sources/test_source_spec.py -v` -Expected: All PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/core/sources/source_spec.py tests/test_core/sources/test_source_spec.py -git commit -m "feat(sources): add SourceSpec typed input-slot class" -``` - ---- - -## Task 3: SourceSpec Exports + ExecutionContext Stub - -**Files:** -- Modify: `src/orcapod/core/sources/__init__.py` -- Create: `src/orcapod/pipeline/execution_context.py` - -- [ ] **Step 1: Add SourceSpec to sources `__init__.py`** - -In `src/orcapod/core/sources/__init__.py`, add after the last existing import: - -```python -from .source_spec import SourceSpec -``` - -And add `"SourceSpec"` to `__all__`. - -- [ ] **Step 2: Create `ExecutionContext` stub** - -```python -# src/orcapod/pipeline/execution_context.py -"""ExecutionContext — stub type for pipeline execution configuration. - -Full definition (PipelineConfig integration, distributed execution) is -deferred to a follow-up issue. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from orcapod.types import PipelineConfig - - -@dataclass(frozen=True) -class ExecutionContext: - """Minimal placeholder for pipeline execution configuration. - - Full definition including ``PipelineConfig`` integration is deferred - to a follow-up issue. - - Args: - config: Optional pipeline-level execution configuration. - """ - - config: "PipelineConfig | None" = None -``` - -- [ ] **Step 3: Verify imports** - -Run: `uv run python -c "from orcapod.core.sources import SourceSpec; from orcapod.pipeline.execution_context import ExecutionContext; print('OK')"` -Expected: `OK` - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/core/sources/__init__.py src/orcapod/pipeline/execution_context.py -git commit -m "feat(pipeline): add ExecutionContext stub; export SourceSpec from sources" -``` - ---- - -## Task 4: Refactor `Pipeline` — Remove DB/Execution, Enforce SourceSpec - -**Files:** -- Modify: `src/orcapod/pipeline/graph.py` - -This task removes all database, execution, and observer logic from `Pipeline`, leaving only DAG recording + compilation, plus a new `bind()` method. The `run()`, `flush()`, `_apply_execution_engine()`, `_compute_pipeline_snapshot_hash()`, and all scoped-DB properties are removed. `compile()` gains SourceSpec-only enforcement. - -- [ ] **Step 1: Write failing tests for the new Pipeline interface** - -Add to `tests/test_pipeline/test_pipeline.py` (before existing classes): - -```python -# At top of file, add import: -from orcapod.core.sources.source_spec import SourceSpec - -class TestPipelineSourceSpecEnforcement: - def test_pipeline_no_database_params(self): - """New Pipeline() takes only name (no pipeline_database).""" - pipeline = Pipeline(name="test") - with pipeline: - pass - assert pipeline._compiled - - def test_pipeline_with_spec_leaves_compiles(self): - """Pipeline with SourceSpec leaves compiles without error.""" - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("input_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("input_b", tag_schema=tag_b, data_schema=data_b) - - pipeline = Pipeline(name="spec_pipe") - with pipeline: - Join()(spec_a, spec_b) - - assert pipeline._compiled - source_nodes = [ - n for n in pipeline._node_graph.nodes() if isinstance(n, SourceNode) - ] - assert len(source_nodes) == 2 - - def test_pipeline_with_concrete_leaf_raises(self): - """Pipeline.compile() raises ValueError if any leaf is not a SourceSpec.""" - src_a, src_b = _make_two_sources() - - pipeline = Pipeline(name="bad_pipe") - with pytest.raises(ValueError, match="SourceSpec"): - with pipeline: - Join()(src_a, src_b) - - def test_pipeline_bind_returns_pipeline_job(self): - """Pipeline.bind() returns a PipelineJob without modifying the pipeline.""" - from orcapod.pipeline.job import PipelineJob - - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) - - pipeline = Pipeline(name="p") - with pipeline: - Join()(spec_a, spec_b) - - db = InMemoryArrowDatabase() - job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=db) - - assert isinstance(job, PipelineJob) - assert job.pipeline is pipeline - assert job.store is db -``` - -- [ ] **Step 2: Run to verify failures** - -Run: `uv run pytest tests/test_pipeline/test_pipeline.py::TestPipelineSourceSpecEnforcement -v` -Expected: FAIL (Pipeline still accepts pipeline_database, no SourceSpec enforcement) - -- [ ] **Step 3: Rewrite `Pipeline.__init__`** - -Replace the current `Pipeline.__init__` (lines 70–103 in `graph.py`) with: - -```python -def __init__( - self, - name: str | tuple[str, ...], - tracker_manager: cp.TrackerManagerProtocol | None = None, - auto_compile: bool = True, -) -> None: - """Initialize a pure computational blueprint pipeline. - - Args: - name: Pipeline name (string or tuple). Used to scope database paths - when the pipeline is run via a ``PipelineJob``. - tracker_manager: Optional tracker manager override. Defaults to - ``DEFAULT_TRACKER_MANAGER``. - auto_compile: If ``True`` (default), ``compile()`` is called - automatically when the context manager exits. - """ - super().__init__(tracker_manager=tracker_manager) - self._node_lut: dict[str, GraphNode] = {} - self._upstreams: dict[str, cp.StreamProtocol] = {} - self._graph_edges: list[tuple[str, str]] = [] - self._hash_graph: "nx.DiGraph" = nx.DiGraph() - self._name = (name,) if isinstance(name, str) else tuple(name) - self._nodes: dict[str, GraphNode] = {} - self._persistent_node_map: dict[str, GraphNode] = {} - self._node_graph: "nx.DiGraph | None" = None - self._auto_compile = auto_compile - self._compiled = False -``` - -Also remove the instance variable declarations for `_pipeline_database`, `_result_database`, `_result_database_scoped`, `_scoped_pipeline_database`, `_status_database`, `_log_database`, `_default_observer`, and `_auto_save_path`. - -Remove the entire `pipeline_database`, `result_database`, `scoped_pipeline_database`, `status_database`, `log_database` property methods. - -- [ ] **Step 4: Update `Pipeline.compile()` — remove DB logic, add SourceSpec enforcement** - -At the start of `compile()`, after `from orcapod.core.nodes import FunctionNode, OperatorNode`, remove all the scoped database creation block (the `if self._pipeline_database is not None:` block through `self._default_observer = NoOpObserver()`). - -Replace the leaf-node wrapping block: -```python -if node_hash not in self._node_lut: - # -- Leaf stream: wrap in SourceNode -- - stream = self._upstreams[node_hash] - node = SourceNode(stream=stream) - persistent_node_map[node_hash] = node -``` - -With the SourceSpec-enforcing version: -```python -if node_hash not in self._node_lut: - # -- Leaf stream: must be a SourceSpec in the new design -- - from orcapod.core.sources.source_spec import SourceSpec - stream = self._upstreams[node_hash] - if not isinstance(stream, SourceSpec): - raise ValueError( - f"Pipeline: all leaf inputs must be SourceSpec instances, " - f"but found {type(stream).__name__!r}. " - "Use 'with PipelineJob:' to record a pipeline with concrete sources, " - "or replace concrete sources with SourceSpec declarations." - ) - node = SourceNode(stream=stream) - persistent_node_map[node_hash] = node -``` - -Also remove the database-attachment blocks inside the `isinstance(node, FunctionNode)` and `isinstance(node, OperatorNode)` branches: - -For FunctionNode — remove: -```python -if pipeline_db is not None: - node.attach_databases( - pipeline_database=pipeline_db, - result_database=result_db, - ) -if node.executor is None: - from orcapod.core.executors.local import LocalPythonFunctionExecutor - node.executor = LocalPythonFunctionExecutor() -``` - -For OperatorNode — remove: -```python -if pipeline_db is not None: - node.attach_databases( - pipeline_database=pipeline_db, - ) -``` - -Remove the import of `NoOpObserver` from the compile() function. - -- [ ] **Step 5: Remove `run()`, `flush()`, `_apply_execution_engine()`, `_compute_pipeline_snapshot_hash()` from Pipeline** - -Delete these methods entirely from `graph.py`. - -- [ ] **Step 6: Add `Pipeline.bind()` stub** - -Add after `compile()`: - -```python -def bind( - self, - sources: "dict[str, cp.StreamProtocol] | None" = None, - store: "dbp.ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, -) -> "PipelineJob": - """Wrap this pipeline in a ``PipelineJob`` with the given bindings. - - Non-mutating — returns a fresh ``PipelineJob``; this ``Pipeline`` - is unchanged. - - Args: - sources: Mapping of SourceSpec name to concrete source. - store: Database for result caching and operator records. - execution_context: Optional execution configuration. - - Returns: - A new ``PipelineJob`` with this pipeline and the given bindings. - """ - from orcapod.pipeline.job import PipelineJob - from orcapod.pipeline.execution_context import ExecutionContext - - return PipelineJob( - _pipeline=self, - sources=sources or {}, - store=store, - execution_context=execution_context, - ) -``` - -Add `ExecutionContext` to the `if TYPE_CHECKING:` block at the top of `graph.py`: -```python -if TYPE_CHECKING: - import networkx as nx - from orcapod.pipeline.execution_context import ExecutionContext - from orcapod.pipeline.serialization import DatabaseRegistry - from orcapod.protocols.database_protocols import DatabaseRegistryProtocol - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol -``` - -- [ ] **Step 7: Run tests** - -Run: `uv run pytest tests/test_pipeline/test_pipeline.py::TestPipelineSourceSpecEnforcement -v` -Expected: All PASS - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/pipeline/graph.py tests/test_pipeline/test_pipeline.py -git commit -m "refactor(pipeline): remove DB/execution surface; enforce SourceSpec-only leaves; add bind()" -``` - ---- - -## Task 5: Update `Pipeline.save()` / `load()` for Pure Blueprint - -**Files:** -- Modify: `src/orcapod/pipeline/graph.py` -- Modify: `src/orcapod/pipeline/serialization.py` -- Modify: `tests/test_pipeline/test_serialization.py` - -The new `Pipeline.save()` always saves at "definition" level (no DB registry). SourceSpec nodes are serialized with `source_type: "spec"`. The old DB-dependent save levels (`"standard"`, `"full"`) move to `PipelineJob.save()`. - -- [ ] **Step 1: Write failing tests for new save format** - -Add to `tests/test_pipeline/test_serialization.py` (replacing or supplementing existing fixtures): - -```python -# New fixture — Pipeline with SourceSpecs (no DB) -@pytest.fixture -def spec_pipeline(tmp_path): - """A compiled Pipeline using SourceSpec leaves.""" - import pyarrow as pa - from orcapod.core.sources import ArrowTableSource - from orcapod.core.sources.source_spec import SourceSpec - from orcapod.core.operators import Join - from orcapod.pipeline import Pipeline - - def _src(tag, data): - tbl = pa.table({tag: pa.array(["a"], type=pa.large_string()), data: pa.array([1], type=pa.int64())}) - return ArrowTableSource(tbl, tag_columns=[tag], infer_nullable=True) - - src_a = _src("key", "value") - src_b = _src("key", "score") - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - - spec_a = SourceSpec("source_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("source_b", tag_schema=tag_b, data_schema=data_b) - - pipeline = Pipeline(name="spec_pipe") - with pipeline: - Join()(spec_a, spec_b, label="joiner") - - return pipeline, tmp_path - - -class TestPipelineBlueprintSave: - def test_save_creates_file(self, spec_pipeline): - pipeline, tmp_path = spec_pipeline - path = tmp_path / "pipeline.json" - pipeline.save(str(path)) - assert path.exists() - - def test_save_has_pipeline_version(self, spec_pipeline): - pipeline, tmp_path = spec_pipeline - import json - path = tmp_path / "pipeline.json" - pipeline.save(str(path)) - data = json.loads(path.read_text()) - assert data["orcapod_pipeline_version"] == "0.1.0" - - def test_save_no_databases_block(self, spec_pipeline): - """Pure blueprint save must not contain a 'databases' block.""" - pipeline, tmp_path = spec_pipeline - import json - path = tmp_path / "pipeline.json" - pipeline.save(str(path)) - data = json.loads(path.read_text()) - assert "databases" not in data - - def test_save_source_spec_nodes(self, spec_pipeline): - """SourceSpec nodes must serialize with source_type='spec'.""" - pipeline, tmp_path = spec_pipeline - import json - path = tmp_path / "pipeline.json" - pipeline.save(str(path)) - data = json.loads(path.read_text()) - spec_nodes = [ - n for n in data["nodes"].values() - if n.get("node_type") == "source" - and n.get("source_config", {}).get("source_type") == "spec" - ] - assert len(spec_nodes) == 2 - - def test_save_load_roundtrip_preserves_topology(self, spec_pipeline): - """load() reconstructs the same number of nodes and edges.""" - pipeline, tmp_path = spec_pipeline - path = tmp_path / "pipeline.json" - pipeline.save(str(path)) - loaded = Pipeline.load(str(path)) - assert len(loaded._persistent_node_map) == len(pipeline._persistent_node_map) - assert len(list(loaded._node_graph.edges())) == len(list(pipeline._node_graph.edges())) - - def test_save_load_restores_spec_names(self, spec_pipeline): - """SourceSpec names must survive save/load.""" - from orcapod.core.sources.source_spec import SourceSpec - pipeline, tmp_path = spec_pipeline - path = tmp_path / "pipeline.json" - pipeline.save(str(path)) - loaded = Pipeline.load(str(path)) - spec_names = { - node.stream.name - for node in loaded._persistent_node_map.values() - if isinstance(node, SourceNode) and isinstance(node.stream, SourceSpec) - } - assert spec_names == {"source_a", "source_b"} -``` - -- [ ] **Step 2: Run tests to confirm failures** - -Run: `uv run pytest tests/test_pipeline/test_serialization.py::TestPipelineBlueprintSave -v` -Expected: FAIL - -- [ ] **Step 3: Simplify `Pipeline.save()`** - -Replace the full `save()` method in `graph.py` with a simplified version: - -```python -def save(self, path: str | Path) -> None: - """Serialize the pure pipeline blueprint to a JSON file. - - Saves topology and SourceSpec declarations only — no databases, - no execution context, no run metadata. - - Args: - path: File path to write JSON output to. - - Raises: - ValueError: If the pipeline has not been compiled. - """ - if not self._compiled: - raise ValueError( - "Pipeline is not compiled. Call compile() or use " - "auto_compile=True before saving." - ) - - import json as _json - from orcapod.pipeline.serialization import ( - PIPELINE_FORMAT_VERSION, - serialize_schema, - ) - from orcapod.core.sources.source_spec import SourceSpec - from orcapod.core.nodes import OperatorNode, FunctionNode - - nodes: dict[str, Any] = {} - for content_hash_str, node in self._persistent_node_map.items(): - tag_schema, data_schema = node.output_schema() - type_converter = node.data_context.type_converter - - descriptor: dict[str, Any] = { - "node_type": node.node_type, - "label": node.label, - "content_hash": node.content_hash().to_string(), - "pipeline_hash": node.pipeline_hash().to_string(), - "output_schema": { - "tag": serialize_schema(tag_schema, type_converter), - "data": serialize_schema(data_schema, type_converter), - }, - "node_uri": list(node.node_uri), - "data_context_key": node.data_context_key, - } - - if isinstance(node, SourceNode): - if isinstance(node.stream, SourceSpec): - descriptor["source_config"] = { - "source_type": "spec", - "spec_name": node.stream.name, - } - descriptor["reconstructable"] = True - else: - descriptor["source_config"] = None - descriptor["reconstructable"] = False - - elif isinstance(node, FunctionNode): - descriptor["function_config"] = node._function_pod.to_config() - descriptor["table_scope"] = node._table_scope - - elif isinstance(node, OperatorNode): - descriptor["operator_config"] = node._operator.to_config() - descriptor["table_scope"] = node._table_scope - - nodes[content_hash_str] = descriptor - - output: dict[str, Any] = { - "orcapod_pipeline_version": PIPELINE_FORMAT_VERSION, - "pipeline": {"name": list(self._name)}, - "nodes": nodes, - "edges": [list(edge) for edge in self._graph_edges], - } - - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - _json.dump(output, f, indent=2) -``` - -Remove the old `_build_source_descriptor`, `_build_function_descriptor`, `_build_operator_descriptor` helper methods (they're replaced by the inline logic above). Keep the `show_graph()` method and `GraphRenderer` / related rendering code unchanged. - -- [ ] **Step 4: Simplify `Pipeline.load()`** - -Replace the `load()` classmethod with a simplified version that only handles blueprint format: - -```python -@classmethod -def load(cls, path: str | Path) -> "Pipeline": - """Deserialize a pure pipeline blueprint from a JSON file. - - Reconstructs topology and SourceSpec declarations. The loaded - pipeline is topology-only — to run it, call - ``pipeline.bind(sources=..., store=...)`` first. - - Args: - path: Path to the JSON file produced by :meth:`save`. - - Returns: - A compiled ``Pipeline`` instance with SourceSpec leaf nodes. - - Raises: - ValueError: If the file's format version is unsupported. - """ - import json as _json - from orcapod.pipeline.serialization import ( - SUPPORTED_FORMAT_VERSIONS, - deserialize_schema, - ) - from orcapod.core.sources.source_spec import SourceSpec - - path = Path(path) - with open(path) as f: - data = _json.load(f) - - version = data.get("orcapod_pipeline_version", "") - if version not in SUPPORTED_FORMAT_VERSIONS: - raise ValueError( - f"Unsupported pipeline format version {version!r}. " - f"Supported: {sorted(SUPPORTED_FORMAT_VERSIONS)}" - ) - - pipeline_meta = data["pipeline"] - name = tuple(pipeline_meta["name"]) - nodes_data = data["nodes"] - edges = data["edges"] - - # Build topological order - edge_graph: "nx.DiGraph" = nx.DiGraph() - for up_hash, down_hash in edges: - edge_graph.add_edge(up_hash, down_hash) - for node_hash in nodes_data: - if node_hash not in edge_graph: - edge_graph.add_node(node_hash) - topo_order = list(nx.topological_sort(edge_graph)) - - upstream_map: dict[str, list[str]] = {} - for up_hash, down_hash in edges: - upstream_map.setdefault(down_hash, []).append(up_hash) - - reconstructed: dict[str, GraphNode] = {} - - for node_hash in topo_order: - descriptor = nodes_data.get(node_hash) - if descriptor is None: - continue - - node_type = descriptor.get("node_type") - source_config = descriptor.get("source_config") or {} - - if node_type == "source": - if source_config.get("source_type") == "spec": - spec_name = source_config["spec_name"] - tag_schema = deserialize_schema(descriptor["output_schema"]["tag"]) - data_schema = deserialize_schema(descriptor["output_schema"]["data"]) - stream = SourceSpec( - name=spec_name, - tag_schema=tag_schema, - data_schema=data_schema, - ) - else: - stream = None # non-spec source, load as read-only stub - - node = SourceNode.from_descriptor(descriptor, stream=stream, databases={}) - reconstructed[node_hash] = node - - elif node_type == "function": - up_hashes = upstream_map.get(node_hash, []) - upstream_node = reconstructed.get(up_hashes[0]) if up_hashes else None - node = FunctionNode.from_descriptor( - descriptor, function_pod=None, input_stream=upstream_node, databases={} - ) - reconstructed[node_hash] = node - - elif node_type == "operator": - up_hashes = upstream_map.get(node_hash, []) - upstream_nodes = tuple( - reconstructed[h] for h in up_hashes if h in reconstructed - ) - node = OperatorNode.from_descriptor( - descriptor, operator=None, input_streams=(), databases={} - ) - reconstructed[node_hash] = node - - # Build Pipeline instance - pipeline = cls(name=name, auto_compile=False) - pipeline._persistent_node_map = dict(reconstructed) - - pipeline._nodes = { - node.label: node - for node in reconstructed.values() - if node.label - } - - pipeline._node_graph = nx.DiGraph() - for up_hash, down_hash in edges: - up_node = reconstructed.get(up_hash) - down_node = reconstructed.get(down_hash) - if up_node is not None and down_node is not None: - pipeline._node_graph.add_edge(up_node, down_node) - for node in reconstructed.values(): - if node not in pipeline._node_graph: - pipeline._node_graph.add_node(node) - - pipeline._graph_edges = [(up, down) for up, down in edges] - pipeline._hash_graph = nx.DiGraph() - for up_hash, down_hash in edges: - pipeline._hash_graph.add_edge(up_hash, down_hash) - for node_hash, node in reconstructed.items(): - if node_hash not in pipeline._hash_graph: - pipeline._hash_graph.add_node(node_hash) - attrs = pipeline._hash_graph.nodes[node_hash] - attrs["node_type"] = node.node_type - if node.label: - attrs["label"] = node.label - - pipeline._compiled = True - return pipeline -``` - -Add `deserialize_schema` to `src/orcapod/pipeline/serialization.py` if it does not already exist. It should reconstruct a `Schema` from the `{"field_name": "type_str"}` format that `serialize_schema` produces. Check the existing code — if `deserialize_schema` is already present, just ensure it is importable. - -- [ ] **Step 5: Run tests** - -Run: `uv run pytest tests/test_pipeline/test_serialization.py::TestPipelineBlueprintSave -v` -Expected: All PASS - -Then run the full serialization suite to see which existing tests are now failing (expected — they test the old format with DBs): - -Run: `uv run pytest tests/test_pipeline/test_serialization.py -v --tb=no -q` - -Note the failing test names — they will be migrated to `test_pipeline_job.py` in Task 10. - -- [ ] **Step 6: Remove static helper methods for old save** - -Delete `_load_source_node`, `_load_function_node`, `_load_operator_node` from `Pipeline` (they were helper statics for the old complex load; the new load is inline). - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/pipeline/graph.py src/orcapod/pipeline/serialization.py \ - tests/test_pipeline/test_serialization.py -git commit -m "refactor(pipeline): simplify save/load to pure blueprint format with SourceSpec support" -``` - ---- - -## Task 6: `PipelineJob` — With-Block Recording, `bind()`, Completeness - -**Files:** -- Create: `src/orcapod/pipeline/job.py` -- Create: `tests/test_pipeline/test_pipeline_job.py` - -- [ ] **Step 1: Write failing tests** - -```python -# tests/test_pipeline/test_pipeline_job.py -from __future__ import annotations - -from typing import cast - -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode -from orcapod.core.operators import Join -from orcapod.core.sources import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec -from orcapod.databases import InMemoryArrowDatabase -from orcapod.errors import SourceSpecMismatchError -from orcapod.pipeline.job import PipelineJob - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_source(tag_col: str, data_col: str, data: dict) -> ArrowTableSource: - table = pa.table( - { - tag_col: pa.array(data[tag_col], type=pa.large_string()), - data_col: pa.array(data[data_col], type=pa.int64()), - } - ) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) - - -def _make_two_sources(): - src_a = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) - src_b = _make_source("key", "score", {"key": ["a", "b"], "score": [100, 200]}) - return src_a, src_b - - -def add_values(value: int, score: int) -> int: - return value + score - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture -def store(): - return InMemoryArrowDatabase() - - -# --------------------------------------------------------------------------- -# Tests: with-block recording -# --------------------------------------------------------------------------- - - -class TestPipelineJobRecording: - def test_with_concrete_sources_auto_creates_specs(self, store): - """Concrete sources in with-block become SourceSpecs in job.pipeline.""" - src_a, src_b = _make_two_sources() - - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - # Pipeline should have SourceSpec leaf nodes - source_nodes = [ - n for n in job.pipeline._node_graph.nodes() if isinstance(n, SourceNode) - ] - assert all(isinstance(n.stream, SourceSpec) for n in source_nodes) - - def test_concrete_source_stored_in_sources(self, store): - """Concrete sources from with-block are stored by label in job.sources.""" - src_a, src_b = _make_two_sources() - src_a._label = "source_a" - src_b._label = "source_b" - - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - assert "source_a" in job.sources - assert "source_b" in job.sources - assert job.sources["source_a"] is src_a - assert job.sources["source_b"] is src_b - - def test_spec_leaf_not_added_to_sources(self, store): - """SourceSpec leaves are NOT added to job.sources (they're unbound).""" - src_a, _ = _make_two_sources() - tag_b, data_b = _make_source("key", "score", {"key": ["a"], "score": [1]}).output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(src_a, spec_b) - - assert "spec_b" not in job.sources - - def test_pipeline_extracted_after_with_block(self, store): - """job.pipeline is a compiled Pipeline after the with block.""" - from orcapod.pipeline.graph import Pipeline - - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - assert isinstance(job.pipeline, Pipeline) - assert job.pipeline._compiled - - -# --------------------------------------------------------------------------- -# Tests: bind() -# --------------------------------------------------------------------------- - - -class TestPipelineJobBind: - def test_bind_sources_returns_new_job(self, store): - """bind(sources=...) returns a new PipelineJob; original is unchanged.""" - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(spec_a, spec_b) - - job2 = job.bind(sources={"a": src_a, "b": src_b}) - assert job2 is not job - assert job.sources == {} # original unchanged - assert "a" in job2.sources and "b" in job2.sources - - def test_bind_store_returns_new_job(self, store): - src_a, src_b = _make_two_sources() - job = PipelineJob() - with job: - Join()(src_a, src_b) - - new_store = InMemoryArrowDatabase() - job2 = job.bind(store=new_store) - assert job2.store is new_store - assert job.store is None # original unchanged - - def test_bind_preserves_existing_sources(self, store): - """bind(sources=...) merges new sources with existing ones.""" - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(spec_a, spec_b) - - job2 = job.bind(sources={"a": src_a}) - job3 = job2.bind(sources={"b": src_b}) - - assert "a" in job3.sources - assert "b" in job3.sources - - def test_bind_validates_schema_at_bind_time(self, store): - """bind() raises SourceSpecMismatchError for incompatible sources.""" - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - # Create a spec that requires an extra column the source doesn't have - from orcapod.types import Schema - wrong_spec = SourceSpec("a", tag_schema=tag_a, data_schema=Schema({"value": int, "extra": str})) - - job = PipelineJob(store=store) - with job: - Join()(wrong_spec, src_b) - - with pytest.raises(SourceSpecMismatchError): - job.bind(sources={"a": src_a}) - - def test_pipeline_bind_wraps_in_job(self, store): - """Pipeline.bind() returns a PipelineJob holding that pipeline.""" - from orcapod.pipeline.graph import Pipeline - - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) - - pipeline = Pipeline(name="p") - with pipeline: - Join()(spec_a, spec_b) - - job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=store) - assert isinstance(job, PipelineJob) - assert job.pipeline is pipeline - - -# --------------------------------------------------------------------------- -# Tests: completeness -# --------------------------------------------------------------------------- - - -class TestPipelineJobCompleteness: - def test_unbound_specs_lists_unbound(self, store): - """unbound_specs() lists SourceSpec names not in job.sources.""" - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(src_a, spec_b) - - unbound = job.unbound_specs() - assert len(unbound) == 1 - assert unbound[0].name == "spec_b" - - def test_unbound_specs_empty_when_all_bound(self, store): - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) # both auto-bound from labels - - assert job.unbound_specs() == [] - - def test_is_complete_true_when_all_bound_with_store(self, store): - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - assert job.is_complete() - - def test_is_complete_false_when_store_missing(self): - src_a, src_b = _make_two_sources() - job = PipelineJob() # no store - with job: - Join()(src_a, src_b) - - assert not job.is_complete() - - def test_is_complete_false_when_specs_unbound(self, store): - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(src_a, spec_b) - - assert not job.is_complete() -``` - -- [ ] **Step 2: Run to verify failures** - -Run: `uv run pytest tests/test_pipeline/test_pipeline_job.py -v --tb=short 2>&1 | head -30` -Expected: ImportError on `orcapod.pipeline.job` - -- [ ] **Step 3: Implement `PipelineJob`** - -```python -# src/orcapod/pipeline/job.py -"""PipelineJob — pipeline + source bindings + execution context.""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any - -from orcapod.core.tracker import AutoRegisteringContextBasedTracker -from orcapod.protocols import core_protocols as cp - -if TYPE_CHECKING: - from orcapod.core.nodes import FunctionNode, GraphNode, OperatorNode, SourceNode - from orcapod.core.sources.source_spec import SourceSpec - from orcapod.pipeline.execution_context import ExecutionContext - from orcapod.pipeline.graph import Pipeline - from orcapod.protocols.database_protocols import ArrowDatabaseProtocol - -logger = logging.getLogger(__name__) - - -class PipelineJob(AutoRegisteringContextBasedTracker): - """Pipeline + source bindings + execution context. - - ``PipelineJob`` is the everyday working object. It is built incrementally: - its ``with``-block records both the DAG structure and any concrete source - bindings simultaneously. Concrete sources are automatically promoted to - ``SourceSpec`` declarations in the underlying ``Pipeline``, with their - concrete instances stored in ``job.sources``. - - After the ``with`` block, ``job.pipeline`` is a fully compiled, pure - ``Pipeline`` (SourceSpec-only leaves). ``job.run()`` executes the - resolvable subgraph — nodes whose upstream SourceSpecs are all bound. - - ``PipelineJob`` can also be created from a ``Pipeline`` via - ``pipeline.bind(sources=..., store=...)`` for the "explicit blueprint" - workflow. - - Args: - store: Database for result caching and operator records. - execution_context: Optional execution configuration. - tracker_manager: Optional tracker manager override. - _pipeline: Internal — pre-built pipeline (used by Pipeline.bind()). - sources: Internal — pre-bound sources (used by Pipeline.bind() / - bind()). - """ - - def __init__( - self, - store: "ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, - tracker_manager: cp.TrackerManagerProtocol | None = None, - *, - _pipeline: "Pipeline | None" = None, - sources: "dict[str, cp.StreamProtocol] | None" = None, - ) -> None: - super().__init__(tracker_manager=tracker_manager) - self._store = store - self._execution_context = execution_context - self._compiled_pipeline: "Pipeline | None" = _pipeline - self._sources: dict[str, cp.StreamProtocol] = dict(sources or {}) - - # Recording state (populated during with-block) - self._rec_graph_edges: list[tuple[str, str]] = [] - self._rec_upstreams: dict[str, cp.StreamProtocol] = {} - self._rec_node_lut: dict[str, "GraphNode"] = {} - self._spec_by_name: dict[str, "SourceSpec"] = {} - self._pipeline_name: tuple[str, ...] = ("pipeline",) - - # ------------------------------------------------------------------ - # Context manager — recording - # ------------------------------------------------------------------ - - def __enter__(self) -> "PipelineJob": - # Reset recording state - self._rec_graph_edges = [] - self._rec_upstreams = {} - self._rec_node_lut = {} - self._spec_by_name = {} - return super().__enter__() # type: ignore[return-value] - - def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: - super().__exit__(exc_type, exc_value, traceback) - if exc_type is None: - self._compile_from_recording() - - def _compile_from_recording(self) -> None: - """Compile the recorded edges into a pure Pipeline.""" - from orcapod.pipeline.graph import Pipeline - - pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) - # Inject the recording state into the pipeline - pipeline._graph_edges = list(self._rec_graph_edges) - pipeline._upstreams = dict(self._rec_upstreams) - pipeline._node_lut = dict(self._rec_node_lut) - # Rebuild hash graph - import orcapod.utils.lazy_module as _lm - import importlib - nx = importlib.import_module("networkx") - for edge in self._rec_graph_edges: - pipeline._hash_graph.add_edge(*edge) - - pipeline.compile() - self._compiled_pipeline = pipeline - - def _ensure_spec(self, source: cp.StreamProtocol) -> "SourceSpec": - """Promote *source* to a SourceSpec, storing the concrete binding. - - If the spec already exists (same label), returns the cached spec. - """ - from orcapod.core.sources.source_spec import SourceSpec - - name = source.label # type: ignore[attr-defined] - if name not in self._spec_by_name: - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name=name, tag_schema=tag_schema, data_schema=data_schema) - self._spec_by_name[name] = spec - self._sources[name] = source - return self._spec_by_name[name] - - @staticmethod - def _is_concrete_source(stream: cp.StreamProtocol) -> bool: - """True if *stream* is a concrete RootSource (not a SourceSpec).""" - from orcapod.core.sources.base import RootSource - from orcapod.core.sources.source_spec import SourceSpec - - return isinstance(stream, RootSource) and not isinstance(stream, SourceSpec) - - # ------------------------------------------------------------------ - # TrackerProtocol — recording with source interception - # ------------------------------------------------------------------ - - def record_function_pod_invocation( - self, - pod: cp.FunctionPodProtocol, - input_stream: cp.StreamProtocol, - label: str | None = None, - ) -> None: - """Record a function pod invocation, promoting concrete sources to specs.""" - from orcapod.core.nodes import FunctionNode - - if self._is_concrete_source(input_stream): - input_stream = self._ensure_spec(input_stream) - - input_hash = input_stream.content_hash().to_string() - function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) - fn_hash = function_node.content_hash().to_string() - - self._rec_node_lut[fn_hash] = function_node - self._rec_upstreams[input_hash] = input_stream - self._rec_graph_edges.append((input_hash, fn_hash)) - - def record_operator_pod_invocation( - self, - pod: cp.OperatorPodProtocol, - upstreams: tuple[cp.StreamProtocol, ...] = (), - label: str | None = None, - ) -> None: - """Record an operator pod invocation, promoting concrete sources to specs.""" - from orcapod.core.nodes import OperatorNode - - processed = tuple( - self._ensure_spec(s) if self._is_concrete_source(s) else s - for s in upstreams - ) - - operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) - op_hash = operator_node.content_hash().to_string() - - self._rec_node_lut[op_hash] = operator_node - for up_hash, upstream in zip( - [s.content_hash().to_string() for s in processed], processed - ): - self._rec_upstreams[up_hash] = upstream - self._rec_graph_edges.append((up_hash, op_hash)) - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def pipeline(self) -> "Pipeline": - """The compiled pure Pipeline (SourceSpec-only leaves). - - Raises: - RuntimeError: If the with-block has not been completed yet. - """ - if self._compiled_pipeline is None: - raise RuntimeError( - "PipelineJob has no compiled pipeline yet. " - "Use 'with job:' to record a DAG first." - ) - return self._compiled_pipeline - - @property - def sources(self) -> dict[str, cp.StreamProtocol]: - """Mapping of SourceSpec name to bound concrete source.""" - return dict(self._sources) - - @property - def store(self) -> "ArrowDatabaseProtocol | None": - """The database used for result caching.""" - return self._store - - @property - def execution_context(self) -> "ExecutionContext | None": - """The execution configuration, or ``None`` if unset.""" - return self._execution_context - - # ------------------------------------------------------------------ - # bind() — non-mutating - # ------------------------------------------------------------------ - - def bind( - self, - sources: "dict[str, cp.StreamProtocol] | None" = None, - store: "ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, - ) -> "PipelineJob": - """Return a new ``PipelineJob`` with updated bindings. - - Non-mutating — the original ``PipelineJob`` is unchanged. Existing - bindings not mentioned in this call are carried forward. - - ``SourceSpec.validate()`` is called for each source in *sources*; - ``SourceSpecMismatchError`` is raised on schema mismatch. - - Args: - sources: Mapping of SourceSpec name → concrete source. Each - source is validated against the matching SourceSpec. - store: Replaces the current store. - execution_context: Replaces the current execution context. - - Returns: - A new ``PipelineJob`` with merged bindings. - - Raises: - SourceSpecMismatchError: If any source's schema is incompatible. - """ - from orcapod.core.nodes import SourceNode - from orcapod.core.sources.source_spec import SourceSpec - - merged_sources = dict(self._sources) - if sources: - # Validate each supplied source against its SourceSpec - pipeline = self._compiled_pipeline - if pipeline is not None: - for node in pipeline._persistent_node_map.values(): - if ( - isinstance(node, SourceNode) - and isinstance(node.stream, SourceSpec) - and node.stream.name in sources - ): - node.stream.validate(sources[node.stream.name]) - merged_sources.update(sources) - - return PipelineJob( - store=store if store is not None else self._store, - execution_context=execution_context if execution_context is not None else self._execution_context, - _pipeline=self._compiled_pipeline, - sources=merged_sources, - ) - - # ------------------------------------------------------------------ - # Completeness introspection - # ------------------------------------------------------------------ - - def unbound_specs(self) -> "list[SourceSpec]": - """Return all SourceSpec slots not yet bound in this job. - - Returns: - List of unbound ``SourceSpec`` instances, in order of appearance - in the pipeline graph. - """ - from orcapod.core.nodes import SourceNode - from orcapod.core.sources.source_spec import SourceSpec - - if self._compiled_pipeline is None: - return [] - - unbound = [] - seen: set[str] = set() - for node in self._compiled_pipeline._persistent_node_map.values(): - if ( - isinstance(node, SourceNode) - and isinstance(node.stream, SourceSpec) - and node.stream.name not in self._sources - and node.stream.name not in seen - ): - unbound.append(node.stream) - seen.add(node.stream.name) - return unbound - - def is_complete(self) -> bool: - """Return ``True`` when all specs are bound and a store is set. - - Returns: - ``True`` if all SourceSpec slots are bound and a store is set. - """ - return self._store is not None and len(self.unbound_specs()) == 0 - - def is_runnable(self, node_label: str) -> bool: - """Return ``True`` if all upstream inputs of *node_label* are resolved. - - Args: - node_label: Label of the node to check. - - Returns: - ``True`` if the node can be executed with current bindings. - """ - from orcapod.core.nodes import SourceNode - from orcapod.core.sources.source_spec import SourceSpec - - pipeline = self._compiled_pipeline - if pipeline is None: - return False - - target = pipeline._nodes.get(node_label) - if target is None: - return False - - import networkx as nx - - for node in nx.ancestors(pipeline._node_graph, target) | {target}: - if ( - isinstance(node, SourceNode) - and isinstance(node.stream, SourceSpec) - and node.stream.name not in self._sources - ): - return False - return True -``` - -- [ ] **Step 4: Run tests** - -Run: `uv run pytest tests/test_pipeline/test_pipeline_job.py -v` -Expected: All PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/pipeline/job.py tests/test_pipeline/test_pipeline_job.py -git commit -m "feat(pipeline): add PipelineJob with with-block recording, source interception, and bind()" -``` - ---- - -## Task 7: `PipelineJob.run()` — Execution Graph + Orchestration - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` -- Modify: `tests/test_pipeline/test_pipeline_job.py` - -- [ ] **Step 1: Write failing tests for run()** - -Append to `tests/test_pipeline/test_pipeline_job.py`: - -```python -class TestPipelineJobRun: - def test_run_executes_all_nodes(self, store): - """run() executes all nodes when all specs are bound.""" - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, src_b) - pod(joined, label="adder") - - job.run() - - node = job.pipeline.compiled_nodes["adder"] - records = node.get_all_records() - assert records is not None - assert records.num_rows == 2 - - def test_run_produces_correct_values(self, store): - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, src_b, label="joiner") - pod(joined, label="adder") - - job.run() - - table = job.pipeline.compiled_nodes["adder"].as_table() - totals = sorted(cast(list[int], table.column("total").to_pylist())) - assert totals == [110, 220] # a: 10+100, b: 20+200 - - def test_run_partial_execution_skips_unbound_subgraph(self, store): - """Nodes with unbound upstream SourceSpecs are excluded from execution.""" - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, spec_b) - pod(joined, label="adder") - - result = job.run() - # Unresolved specs should be reported - assert "spec_b" in result.unresolved_specs - - def test_run_is_non_mutating(self, store): - """run() returns a new PipelineJob; original is unchanged.""" - src_a, src_b = _make_two_sources() - - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - result = job.run() - assert result is not job - - def test_run_requires_store(self): - """run() without a store raises ValueError.""" - src_a, src_b = _make_two_sources() - - job = PipelineJob() # no store - with job: - Join()(src_a, src_b) - - with pytest.raises(ValueError, match="store"): - job.run() - - -class TestPipelineJobEndToEnd: - def test_end_to_end_source_join_function(self, store): - """Two sources → Join → FunctionPod all execute correctly.""" - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, src_b, label="joiner") - pod(joined, label="adder") - - assert isinstance(job.pipeline.compiled_nodes["joiner"], OperatorNode) - assert isinstance(job.pipeline.compiled_nodes["adder"], FunctionNode) - - job.run() - - fn_records = job.pipeline.compiled_nodes["adder"].get_all_records() - assert fn_records is not None - assert fn_records.num_rows == 2 - - table = job.pipeline.compiled_nodes["adder"].as_table() - totals = sorted(cast(list[int], table.column("total").to_pylist())) - assert totals == [110, 220] - - def test_bind_then_run(self, store): - """Pipeline.bind() + job.run() produces correct results.""" - from orcapod.pipeline.graph import Pipeline - - src_a, src_b = _make_two_sources() - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("src_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("src_b", tag_schema=tag_b, data_schema=data_b) - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - pipeline = Pipeline(name="bp") - with pipeline: - joined = Join()(spec_a, spec_b) - pod(joined, label="adder") - - job = pipeline.bind( - sources={"src_a": src_a, "src_b": src_b}, - store=store, - ) - job.run() - - table = job.pipeline.compiled_nodes["adder"].as_table() - totals = sorted(cast(list[int], table.column("total").to_pylist())) - assert totals == [110, 220] -``` - -- [ ] **Step 2: Run to confirm failures** - -Run: `uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobRun -v` -Expected: FAIL (AttributeError: `PipelineJob` has no `run` method) - -- [ ] **Step 3: Implement `PipelineJob._build_execution_graph()` and `run()`** - -Add to `src/orcapod/pipeline/job.py`: - -```python -# At top of file, add to imports: -from orcapod.utils.lazy_module import LazyModule -# ... (in the else branch for TYPE_CHECKING) -# Add: nx = LazyModule("networkx") - -def _build_execution_graph( - self, -) -> "tuple[Any, list[str], ArrowDatabaseProtocol, ArrowDatabaseProtocol]": - """Build a fresh execution-ready graph with concrete sources substituted. - - Creates new SourceNode/FunctionNode/OperatorNode objects — does NOT - mutate the shared ``pipeline._persistent_node_map`` nodes. - - Returns: - Tuple of (exec_graph, unresolved_spec_names, pipeline_db, result_db). - - Raises: - ValueError: If ``self.store`` is ``None``. - """ - import networkx as nx - from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode - from orcapod.core.sources.source_spec import SourceSpec - - pipeline = self._compiled_pipeline - if pipeline is None: - raise ValueError("No compiled pipeline — use 'with job:' first.") - - store = self._store - if store is None: - raise ValueError( - "PipelineJob.run() requires a store. " - "Call job.bind(store=db) before run()." - ) - - pipeline_db = store.at(*pipeline.name) - result_db = pipeline_db.at("_result") - - # Build the raw recording graph (hash-based) - G: "nx.DiGraph" = nx.DiGraph() - for edge in pipeline._graph_edges: - G.add_edge(*edge) - - exec_node_map: dict[str, GraphNode] = {} - excluded_hashes: set[str] = set() - unresolved_specs: list[str] = [] - - for node_hash in nx.topological_sort(G): - if node_hash in excluded_hashes: - continue - - if node_hash not in pipeline._node_lut: - # Leaf stream - stream = pipeline._upstreams.get(node_hash) - if isinstance(stream, SourceSpec): - if stream.name in self._sources: - # Bound — use concrete source - exec_node_map[node_hash] = SourceNode( - stream=self._sources[stream.name] - ) - else: - # Unbound — exclude this branch - excluded_hashes.add(node_hash) - if stream.name not in unresolved_specs: - unresolved_specs.append(stream.name) - elif stream is not None: - exec_node_map[node_hash] = SourceNode(stream=stream) - else: - template = pipeline._node_lut[node_hash] - preds = list(G.predecessors(node_hash)) - - if any(p in excluded_hashes for p in preds): - excluded_hashes.add(node_hash) - continue - - if isinstance(template, FunctionNode): - assert len(preds) == 1, "FunctionNode must have exactly one upstream" - input_node = exec_node_map[preds[0]] - new_fn = FunctionNode( - function_pod=template._function_pod, - input_stream=input_node, - label=template._label, - ) - new_fn.attach_databases( - pipeline_database=pipeline_db, - result_database=result_db, - ) - # Preserve any executor setting from compile - from orcapod.core.executors.local import LocalPythonFunctionExecutor - if template.executor is not None: - new_fn.executor = template.executor - else: - new_fn.executor = LocalPythonFunctionExecutor() - exec_node_map[node_hash] = new_fn - - elif isinstance(template, OperatorNode): - upstream_nodes = tuple(exec_node_map[p] for p in preds) - new_op = OperatorNode( - operator=template._operator, - input_streams=upstream_nodes, - label=template._label, - ) - new_op.attach_databases(pipeline_database=pipeline_db) - exec_node_map[node_hash] = new_op - - # Build execution graph - exec_graph: "nx.DiGraph" = nx.DiGraph() - for up_hash, down_hash in pipeline._graph_edges: - if up_hash in exec_node_map and down_hash in exec_node_map: - exec_graph.add_edge(exec_node_map[up_hash], exec_node_map[down_hash]) - for node in exec_node_map.values(): - if node not in exec_graph: - exec_graph.add_node(node) - - return exec_graph, unresolved_specs, pipeline_db, result_db - - -def run( - self, - observer: "ExecutionObserverProtocol | None" = None, -) -> "PipelineJob": - """Execute the resolvable subgraph. - - Nodes whose upstream includes an unbound SourceSpec (and all their - dependents) are excluded. Partial execution is a first-class outcome, - not an error — excluded spec names are recorded in the returned job's - ``unresolved_specs``. - - Args: - observer: Optional execution observer. - - Returns: - A new ``PipelineJob`` with run metadata populated (does not mutate - ``self``). - - Raises: - ValueError: If no store is set. - RuntimeError: If no pipeline has been recorded. - """ - import hashlib - - from orcapod.pipeline.observer import NoOpObserver - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - - exec_graph, unresolved_specs, pipeline_db, result_db = self._build_execution_graph() - - effective_observer = observer or NoOpObserver() - - # Compute snapshot hash for run URI - node_strs = sorted(str(n.content_hash().to_string()) for n in exec_graph.nodes()) - snapshot_hash = hashlib.sha256("\n".join(node_strs).encode()).hexdigest()[:16] - pipeline_uri = "/".join(self._compiled_pipeline.name) + "@" + snapshot_hash - - SyncPipelineOrchestrator().run( - exec_graph, observer=effective_observer, pipeline_uri=pipeline_uri - ) - - # Flush databases - pipeline_db.flush() - result_db.flush() - - # Return new job with run metadata - result = PipelineJob( - store=self._store, - execution_context=self._execution_context, - _pipeline=self._compiled_pipeline, - sources=dict(self._sources), - ) - result._unresolved_specs = unresolved_specs - return result -``` - -Also add a `_unresolved_specs` property: - -```python -@property -def unresolved_specs(self) -> list[str]: - """Spec names that were unbound at run time (excluded from execution).""" - return list(getattr(self, "_unresolved_specs", [])) -``` - -Add `ExecutionObserverProtocol` to the `if TYPE_CHECKING:` block at the top. - -- [ ] **Step 4: Run tests** - -Run: `uv run pytest tests/test_pipeline/test_pipeline_job.py -v` -Expected: All PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/pipeline/job.py tests/test_pipeline/test_pipeline_job.py -git commit -m "feat(pipeline): add PipelineJob.run() with partial execution support" -``` - ---- - -## Task 8: `PipelineJob.save()` / `load()` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` -- Modify: `src/orcapod/pipeline/serialization.py` -- Modify: `tests/test_pipeline/test_pipeline_job.py` - -- [ ] **Step 1: Add `PIPELINE_JOB_FORMAT_VERSION` to `serialization.py`** - -Append to `src/orcapod/pipeline/serialization.py`: - -```python -# --------------------------------------------------------------------------- -# PipelineJob format version -# --------------------------------------------------------------------------- - -PIPELINE_JOB_FORMAT_VERSION = "0.1.0" -SUPPORTED_JOB_FORMAT_VERSIONS = frozenset({"0.1.0"}) -``` - -- [ ] **Step 2: Write failing tests** - -Append to `tests/test_pipeline/test_pipeline_job.py`: - -```python -class TestPipelineJobSerialization: - def test_save_creates_file(self, store, tmp_path): - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - path = tmp_path / "job.json" - job.save(str(path)) - assert path.exists() - - def test_save_has_version(self, store, tmp_path): - import json - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - path = tmp_path / "job.json" - job.save(str(path)) - data = json.loads(path.read_text()) - assert data["orcapod_pipeline_job_version"] == "0.1.0" - - def test_save_includes_run_block(self, store, tmp_path): - """Unsaved template has status=pending and null run fields.""" - import json - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - path = tmp_path / "job.json" - job.save(str(path)) - data = json.loads(path.read_text()) - assert data["run"]["status"] == "pending" - assert data["run"]["run_id"] is None - - def test_load_roundtrip_restores_pipeline(self, store, tmp_path): - """load() restores the pipeline topology.""" - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b, label="joiner") - - path = tmp_path / "job.json" - job.save(str(path)) - loaded = PipelineJob.load(str(path)) - - assert "joiner" in loaded.pipeline.compiled_nodes - - def test_load_roundtrip_after_run(self, store, tmp_path): - """Save after run() → load → re-run produces correct results.""" - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, src_b) - pod(joined, label="adder") - - completed = job.run() - path = tmp_path / "completed.json" - completed.save(str(path)) - - # Load and re-run with same store - loaded = PipelineJob.load(str(path), store=store) - loaded.run() - - table = loaded.pipeline.compiled_nodes["adder"].as_table() - totals = sorted(cast(list[int], table.column("total").to_pylist())) - assert totals == [110, 220] -``` - -- [ ] **Step 3: Run to confirm failures** - -Run: `uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobSerialization -v` -Expected: FAIL (AttributeError: PipelineJob has no `save`) - -- [ ] **Step 4: Implement `PipelineJob.save()` and `PipelineJob.load()`** - -Add to `src/orcapod/pipeline/job.py`: - -```python -def save(self, path: str | "Path") -> None: - """Serialize this job to a JSON file. - - Saves topology (via the embedded Pipeline) plus bindings metadata and - run results. The format covers both "template" (pre-run) and - "run record" (post-run) states — distinguished by the ``run.status`` - field. - - Args: - path: File path to write JSON output to. - """ - import json as _json - from pathlib import Path as _Path - from orcapod.pipeline.serialization import PIPELINE_JOB_FORMAT_VERSION - - pipeline = self._compiled_pipeline - if pipeline is None: - raise ValueError("No compiled pipeline to save.") - - path = _Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - - # Save the pipeline blueprint inline - import tempfile, os - with tempfile.NamedTemporaryFile(suffix=".json", delete=False, mode="w") as tmp: - tmp_path = tmp.name - try: - pipeline.save(tmp_path) - with open(tmp_path) as f: - pipeline_data = _json.load(f) - finally: - os.unlink(tmp_path) - - # Serialize source configs - sources_block: dict = {} - for spec_name, source in self._sources.items(): - if hasattr(source, "to_config"): - sources_block[spec_name] = source.to_config() - else: - sources_block[spec_name] = {"source_type": "unknown"} - - # Store config - store_block = None - if self._store is not None and hasattr(self._store, "to_config"): - store_block = self._store.to_config() - - output = { - "orcapod_pipeline_job_version": PIPELINE_JOB_FORMAT_VERSION, - "run": { - "run_id": getattr(self, "_run_id", None), - "status": "pending" if not hasattr(self, "_unresolved_specs") else "complete", - "unresolved_specs": getattr(self, "_unresolved_specs", []), - }, - "pipeline": pipeline_data, - "bindings": { - "sources": sources_block, - "store": store_block, - }, - } - - with open(path, "w") as f: - _json.dump(output, f, indent=2) - - -@classmethod -def load( - cls, - path: str | "Path", - store: "ArrowDatabaseProtocol | None" = None, -) -> "PipelineJob": - """Deserialize a ``PipelineJob`` from a JSON file. - - The embedded ``Pipeline`` blueprint is always restored. Concrete source - bindings are restored only for reconstructable source types (CSV, - Delta Lake, etc.). Pass *store* explicitly to override any serialized - store configuration. - - Args: - path: Path to the JSON file produced by :meth:`save`. - store: Optional store override. When provided, takes precedence - over any store configuration in the file. - - Returns: - A ``PipelineJob`` ready to run (pending a ``bind()`` if sources - are missing). - """ - import json as _json - from pathlib import Path as _Path - from orcapod.pipeline.graph import Pipeline - from orcapod.pipeline.serialization import ( - SUPPORTED_JOB_FORMAT_VERSIONS, - resolve_source_from_config, - ) - - path = _Path(path) - with open(path) as f: - data = _json.load(f) - - version = data.get("orcapod_pipeline_job_version", "") - if version not in SUPPORTED_JOB_FORMAT_VERSIONS: - raise ValueError( - f"Unsupported PipelineJob format version {version!r}. " - f"Supported: {sorted(SUPPORTED_JOB_FORMAT_VERSIONS)}" - ) - - # Reconstruct pipeline from embedded blueprint data - import tempfile, os - with tempfile.NamedTemporaryFile(suffix=".json", delete=False, mode="w") as tmp: - tmp_path = tmp.name - _json.dump(data["pipeline"], tmp) - try: - pipeline = Pipeline.load(tmp_path) - finally: - os.unlink(tmp_path) - - # Reconstruct sources - sources: dict[str, cp.StreamProtocol] = {} - for spec_name, src_config in data.get("bindings", {}).get("sources", {}).items(): - try: - source = resolve_source_from_config(src_config) - if source is not None: - sources[spec_name] = source - except Exception: - logger.warning( - "Could not reconstruct source %r from config — skipping.", spec_name - ) - - # Reconstruct store - effective_store = store - if effective_store is None: - store_config = data.get("bindings", {}).get("store") - if store_config: - try: - from orcapod.pipeline.serialization import resolve_database_from_config - effective_store = resolve_database_from_config(store_config) - except Exception: - logger.warning("Could not reconstruct store from config — skipping.") - - job = cls( - store=effective_store, - _pipeline=pipeline, - sources=sources, - ) - - # Restore run metadata - run_block = data.get("run", {}) - if run_block.get("status") != "pending": - job._unresolved_specs = run_block.get("unresolved_specs", []) - - return job -``` - -- [ ] **Step 5: Run tests** - -Run: `uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobSerialization -v` -Expected: All PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/job.py src/orcapod/pipeline/serialization.py \ - tests/test_pipeline/test_pipeline_job.py -git commit -m "feat(pipeline): add PipelineJob.save() / load() serialization" -``` - ---- - -## Task 9: Update All Exports - -**Files:** -- Modify: `src/orcapod/pipeline/__init__.py` -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Update `src/orcapod/pipeline/__init__.py`** - -```python -from .async_orchestrator import AsyncPipelineOrchestrator -from .composite_observer import CompositeObserver -from .execution_context import ExecutionContext -from .graph import Pipeline -from .job import PipelineJob -from .logging_observer import LoggingObserver, DataLogger -from .serialization import ( - LoadStatus, - PIPELINE_FORMAT_VERSION, - PIPELINE_JOB_FORMAT_VERSION, -) -from .status_observer import StatusObserver -from .sync_orchestrator import SyncPipelineOrchestrator - -__all__ = [ - "AsyncPipelineOrchestrator", - "CompositeObserver", - "ExecutionContext", - "LoadStatus", - "LoggingObserver", - "DataLogger", - "PIPELINE_FORMAT_VERSION", - "PIPELINE_JOB_FORMAT_VERSION", - "Pipeline", - "PipelineJob", - "StatusObserver", - "SyncPipelineOrchestrator", -] -``` - -- [ ] **Step 2: Update `src/orcapod/__init__.py`** - -```python -from .core.function_pod import FunctionPod, function_pod -from .core.sources.source_spec import SourceSpec -from .pipeline import Pipeline, PipelineJob - -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -__all__ = [ - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceSpec", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", -] -``` - -- [ ] **Step 3: Verify top-level imports** - -Run: `uv run python -c "from orcapod import Pipeline, PipelineJob, SourceSpec; print('OK')"` -Expected: `OK` - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/pipeline/__init__.py src/orcapod/__init__.py -git commit -m "chore(exports): add PipelineJob, SourceSpec, ExecutionContext to public API" -``` - ---- - -## Task 10: Migrate `test_pipeline.py` - -**Files:** -- Modify: `tests/test_pipeline/test_pipeline.py` - -The old `Pipeline` accepted `pipeline_database` and had `run()`. All tests that relied on those must migrate to `PipelineJob`. Tests that only test structural compilation (wrapping leaves, function nodes, operator nodes, labels) can stay — updated to use the new `Pipeline` (no DB, with SourceSpec leaves). - -- [ ] **Step 1: Update all fixtures — replace `Pipeline(name=..., pipeline_database=...)` with `PipelineJob(store=...)`** - -For every test that calls `Pipeline.run()`, moves to a `PipelineJob`. The pattern is: - -```python -# BEFORE (old pattern): -pipeline = Pipeline(name="test", pipeline_database=pipeline_db) -with pipeline: - joined = Join()(src_a, src_b) - pod(joined, label="adder") -pipeline.run() -records = pipeline.adder.get_all_records() - -# AFTER (new pattern): -from orcapod.pipeline.job import PipelineJob -job = PipelineJob(store=pipeline_db) -with job: - joined = Join()(src_a, src_b) - pod(joined, label="adder") -job.run() -records = job.pipeline.compiled_nodes["adder"].get_all_records() -``` - -For structural tests (TestCompileSourceWrapping, TestCompileFunctionNode, TestCompileOperatorNode, TestCompileMutatesNodes, TestLabelAccess) that only test `_compiled`, `compiled_nodes`, and node types — keep them but switch to SourceSpec-based leaves: - -```python -# BEFORE: -pipeline = Pipeline(name="test_pipe", pipeline_database=pipeline_db) -with pipeline: - _ = Join()(src_a, src_b) - -# AFTER — Pipeline is now pure, must use PipelineJob for execution: -from orcapod.pipeline.job import PipelineJob -job = PipelineJob(store=pipeline_db) -with job: - _ = Join()(src_a, src_b) -pipeline = job.pipeline # pure Pipeline for structural assertions -``` - -- [ ] **Step 2: Remove `pipeline_db` and `function_db` fixtures from `test_pipeline.py` test classes that migrate to PipelineJob** - -The `pipeline_db` fixture remains for the `store=` parameter of `PipelineJob`. Remove the `result_database=function_db` patterns since `PipelineJob` auto-scopes the result DB. - -- [ ] **Step 3: Migrate `TestAutoCompileAndRun` → `TestPipelineJobRun`** - -Migrate the following tests from `test_pipeline.py` to `test_pipeline_job.py`: -- `test_run_executes_all_nodes` → already added in Task 7 -- `test_run_auto_saves_when_path_set` → update for `PipelineJob` (use `job.save()` after `job.run()`) -- `test_run_does_not_save_when_path_not_set` → remove (auto_save_path is a `PipelineJob`-level concern, not Pipeline) -- `test_auto_save_path_without_database_raises` → remove (no longer applies) - -- [ ] **Step 4: Migrate `TestFunctionDatabaseHandling` → `TestPipelineJobDatabaseHandling`** - -```python -# In test_pipeline_job.py: -class TestPipelineJobDatabaseHandling: - def test_result_database_scoped_to_pipeline_name(self): - """The result DB is auto-scoped to pipeline_name/_result.""" - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - store = InMemoryArrowDatabase() - - job = PipelineJob(store=store) - job._pipeline_name = ("my_pipe",) # set name before with block - with job: - joined = Join()(src_a, src_b) - pod(joined, label="adder") - - # Run to trigger database creation - job.run() - - # After run, the result DB should have been scoped to my_pipe/_result - # Verify via the function node's pipeline_database path - exec_nodes = [ - n for n in job.pipeline.compiled_nodes.values() - if isinstance(n, FunctionNode) - ] - assert len(exec_nodes) > 0 -``` - -- [ ] **Step 5: Migrate `TestEndToEnd` → add to `TestPipelineJobEndToEnd`** - -Already done in Task 7 (`test_end_to_end_source_join_function`). - -- [ ] **Step 6: Update `TestPipelineExtension`** - -The pipeline extension tests use `pipeline.adder` to extend the pipeline in a second `with` block. With the new design, re-entering a `PipelineJob` `with` block should work since `_compile_from_recording` merges with existing state. - -Update the extension tests to use `PipelineJob`: - -```python -class TestPipelineJobExtension: - def test_extend_pipeline_with_new_sources(self): - """Re-enter job context to extend the pipeline.""" - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - store = InMemoryArrowDatabase() - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, src_b, label="joiner") - - # Extend in a second with block - src_c = _make_source("key", "extra", {"key": ["a", "b"], "extra": [1000, 2000]}) - # ... (implement extension logic) -``` - -Note: Pipeline extension (re-entering `with` blocks) may need special handling in `PipelineJob._compile_from_recording()` to merge recording state from multiple with-blocks. If the existing pipeline recording's `_hash_graph` accumulation mechanism works correctly (it already does in `Pipeline`), this should work transparently. If not, note it as a follow-up. - -- [ ] **Step 7: Run the full test suite** - -Run: `uv run pytest tests/test_pipeline/test_pipeline.py tests/test_pipeline/test_pipeline_job.py -v --tb=short 2>&1 | tail -30` -Expected: All new tests PASS; note any remaining failures for Task 11. - -- [ ] **Step 8: Commit** - -```bash -git add tests/test_pipeline/test_pipeline.py tests/test_pipeline/test_pipeline_job.py -git commit -m "test(pipeline): migrate execution tests from Pipeline to PipelineJob" -``` - ---- - -## Task 11: Migrate `test_serialization.py` - -**Files:** -- Modify: `tests/test_pipeline/test_serialization.py` - -The serialization tests fall into two categories: - -1. **Tests that use `Pipeline` structurally** (no DB, no run): update to use SourceSpec-based fixtures. -2. **Tests that do DB round-trips** (save → run → load → run): migrate to `test_pipeline_job.py`. - -- [ ] **Step 1: Create new fixture for test_serialization.py** - -Replace the existing `simple_pipeline` and `multi_source_pipeline` fixtures (which use `pipeline_database=...`) with SourceSpec-based versions: - -```python -# In test_serialization.py, update fixtures: - -@pytest.fixture -def simple_pipeline(tmp_path): - """A compiled Pipeline using SourceSpec leaves (no DB).""" - from orcapod.core.sources import ArrowTableSource - from orcapod.core.sources.source_spec import SourceSpec - from orcapod.core.data_function import PythonDataFunction - from orcapod.core.function_pod import FunctionPod - from orcapod.core.operators import Join - from orcapod.pipeline import Pipeline - import pyarrow as pa - - def _src(tag, data_col, vals): - tbl = pa.table({ - tag: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array(vals, type=pa.int64()), - }) - return ArrowTableSource(tbl, tag_columns=[tag], infer_nullable=True) - - src_a = _src("key", "value", [10, 20]) - tag_a, data_a = src_a.output_schema() - - def double(value: int) -> int: - return value * 2 - - pf = PythonDataFunction(double, output_keys="doubled") - pod = FunctionPod(data_function=pf) - spec_a = SourceSpec("input_a", tag_schema=tag_a, data_schema=data_a) - - pipeline = Pipeline(name="simple") - with pipeline: - pod(spec_a, label="doubler") - - path = tmp_path / "simple.json" - pipeline.save(str(path)) - return path, pipeline - - -@pytest.fixture -def multi_source_pipeline(tmp_path): - """A compiled Pipeline with two SourceSpec leaves and a Join.""" - from orcapod.core.sources import ArrowTableSource - from orcapod.core.sources.source_spec import SourceSpec - from orcapod.core.operators import Join - from orcapod.pipeline import Pipeline - import pyarrow as pa - - def _src(tag, data_col): - tbl = pa.table({ - tag: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array([1, 2], type=pa.int64()), - }) - return ArrowTableSource(tbl, tag_columns=[tag], infer_nullable=True) - - src_a = _src("key", "value") - src_b = _src("key", "score") - tag_a, data_a = src_a.output_schema() - tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("source_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("source_b", tag_schema=tag_b, data_schema=data_b) - - pipeline = Pipeline(name="multi") - with pipeline: - Join()(spec_a, spec_b, label="joiner") - - path = tmp_path / "multi.json" - pipeline.save(str(path)) - return path, pipeline -``` - -- [ ] **Step 2: Update structural save tests to use new fixtures** - -For tests in `TestSave` that test basic JSON structure (file creation, version, nodes, edges), update to use the new `spec_pipeline` / `simple_pipeline` fixtures. Remove tests that assert on `"databases"` block presence (it no longer exists). - -- [ ] **Step 3: Delete or migrate DB round-trip tests** - -Tests like `test_save_load_run_full_cycle`, `test_standard_save_load_run_roundtrip`, `test_definition_save_load_run_roundtrip`, and all tests that combine `pipeline.save(level="standard")` with DB setup — these move to `test_pipeline_job.py` under `TestPipelineJobSerialization`. - -The key migration (already done in Task 8): -```python -# In test_pipeline_job.py — already implemented in Task 8: -class TestPipelineJobSerialization: - def test_load_roundtrip_after_run(self, store, tmp_path): ... -``` - -Add more complex round-trip tests to `TestPipelineJobSerialization` as needed, adapting the key scenarios from the old serialization tests. - -- [ ] **Step 4: Run the serialization tests** - -Run: `uv run pytest tests/test_pipeline/test_serialization.py -v --tb=short 2>&1 | tail -30` -Expected: Passing tests are the new structural tests. Remove or skip tests that test removed behavior (old levels, DB registry, etc.). - -- [ ] **Step 5: Run the full test suite** - -Run: `uv run pytest tests/ -v --tb=short -q 2>&1 | tail -40` -Fix any remaining failures. - -- [ ] **Step 6: Commit** - -```bash -git add tests/test_pipeline/test_serialization.py tests/test_pipeline/test_pipeline_job.py -git commit -m "test(pipeline): migrate DB round-trip serialization tests to test_pipeline_job" -``` - ---- - -## Task 12: Final Cleanup and PR - -- [ ] **Step 1: Run the complete test suite** - -Run: `uv run pytest tests/ -v -q 2>&1 | tail -20` -Expected: All tests PASS (or clearly explained skip/xfail). - -- [ ] **Step 2: Check DESIGN_ISSUES.md** - -Read `DESIGN_ISSUES.md` at the project root. Update any open issues that ENG-456 resolves. The old `Pipeline` conflating DAG and execution was a known design issue — mark it as resolved. - -- [ ] **Step 3: Update DESIGN_ISSUES.md for the ExecutionContext follow-up** - -If not already present, add an open issue: - -```markdown -## [open] ExecutionContext full definition - -**Description:** `ExecutionContext` is currently a stub dataclass. Full definition -including `PipelineConfig` integration and distributed execution support is deferred. - -**Related issue:** ENG-456 (parent), follow-up issue TBD. -``` - -- [ ] **Step 4: Push and create PR** - -```bash -git push -u origin eywalker/eng-456-refactor-pipeline-into-source-agnostic-dag-introduce -``` - -Create PR via `gh pr create` targeting `dev` branch with body referencing `Fixes ENG-456`. - ---- - -## Self-Review Checklist - -After writing this plan, verify against the spec: - -**Spec coverage:** -- [x] `SourceSpec` type: Task 2 -- [x] `PipelineJob` type: Tasks 6, 7, 8 -- [x] Refactor `Pipeline` (remove execution surface, enforce SourceSpec): Task 4 -- [x] `with PipelineJob:` recording + auto-SourceSpec: Task 6 -- [x] `job.pipeline` extraction: Task 6 (property) -- [x] `bind()` on both `Pipeline` and `PipelineJob`: Tasks 4 (bind stub), 6 (PipelineJob.bind) -- [x] Partial/progressive execution: Task 7 -- [x] `PipelineJob` save/load: Task 8 -- [x] `Pipeline` save/load as pure blueprint: Task 5 -- [x] Hashing correctness: Task 2 (`test_pipeline_hash_matches_compatible_source`) -- [x] `ExecutionContext` stub: Task 3 -- [x] Test migration: Tasks 10, 11 - -**Type consistency:** -- `SourceSpec.name` → `str` (used as `job.sources` key) -- `PipelineJob.sources` → `dict[str, StreamProtocol]` (always returns a copy) -- `PipelineJob.unbound_specs()` → `list[SourceSpec]` (not list[str]) -- `PipelineJob.unresolved_specs` (property on returned job) → `list[str]` (spec names that were excluded at run time — distinct from `unbound_specs()` which checks current bindings) -- `PipelineJob.bind()` → `PipelineJob` (non-mutating) -- `Pipeline.bind()` → `PipelineJob` (wraps self) -- `Pipeline.save(path)` → no `level` parameter in new design -- `Pipeline.load(path)` → `Pipeline` (no `mode` or DB params) diff --git a/superpowers/plans/2026-05-20-polling-source.md b/superpowers/plans/2026-05-20-polling-source.md deleted file mode 100644 index 80bd33176..000000000 --- a/superpowers/plans/2026-05-20-polling-source.md +++ /dev/null @@ -1,1847 +0,0 @@ -# Polling Source Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement `PollingSource[T]` — a protocol-based dynamic data source that continuously emits rows to async pipelines using a user-supplied `DynamicSourceProtocol[T]` implementation. - -**Architecture:** `DynamicSourceProtocol[T]` supplies three async methods (`poll`, `fetch`, `close`). `PollingSource[T]` wraps it as a `RootSource`, routing sync access through a lazy-initialized `ArrowTableStream` cache and async access through an `async_iter_data()` polling loop. `StreamBase.async_iter_data()` gains a real default (wrapping `iter_data()`), allowing `SourceNode.async_execute` to be simplified to always call `async_iter_data()`. - -**Tech Stack:** Python 3.11+, asyncio, polars, pyarrow, pytest-asyncio - ---- - -## File Map - -| Action | Path | What changes | -|---|---|---| -| Modify | `src/orcapod/types.py` | Add `T` TypeVar, `Cursor[T]` dataclass, `PollingConfig` frozen dataclass | -| Modify | `src/orcapod/errors.py` | Add `CursorInvalidatedError` | -| Modify | `src/orcapod/protocols/core_protocols/sources.py` | Add `DynamicSourceProtocol[T]` | -| Modify | `src/orcapod/protocols/core_protocols/__init__.py` | Export `DynamicSourceProtocol` | -| Modify | `src/orcapod/core/streams/base.py` | Replace `async_iter_data` stub with real default | -| Modify | `src/orcapod/core/nodes/source_node.py` | Simplify `async_execute` to use `async_iter_data()` | -| Create | `src/orcapod/core/sources/polling_source.py` | `PollingSource[T]` class | -| Modify | `src/orcapod/core/sources/__init__.py` | Export `PollingSource` | -| Create | `tests/test_channels/test_polling_source.py` | All tests (19 cases) | - ---- - -## Task 1: New Shared Types — `Cursor[T]`, `PollingConfig`, `CursorInvalidatedError` - -**Files:** -- Modify: `src/orcapod/types.py` -- Modify: `src/orcapod/errors.py` -- Test: `tests/test_channels/test_polling_source.py` (create) - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_channels/test_polling_source.py`: - -```python -"""Tests for PollingSource and DynamicSourceProtocol.""" -from __future__ import annotations - -import asyncio -import logging -from datetime import datetime, timezone -from typing import Any - -import pyarrow as pa -import pytest - -from orcapod.errors import CursorInvalidatedError -from orcapod.types import Cursor, PollingConfig - - -# =========================================================================== -# Task 1: Type tests -# =========================================================================== - - -class TestCursor: - def test_cursor_stores_value(self): - c = Cursor(value=42) - assert c.value == 42 - - def test_cursor_modified_at_defaults_to_none(self): - c = Cursor(value="tok") - assert c.modified_at is None - - def test_cursor_accepts_datetime_modified_at(self): - ts = datetime(2026, 1, 1, tzinfo=timezone.utc) - c = Cursor(value=ts, modified_at=ts) - assert c.modified_at == ts - - def test_cursor_generic_int(self): - c: Cursor[int] = Cursor(value=7) - assert c.value == 7 - - def test_cursor_generic_str(self): - c: Cursor[str] = Cursor(value="page_token") - assert c.value == "page_token" - - -class TestPollingConfig: - def test_defaults(self): - cfg = PollingConfig() - assert cfg.interval == 1.0 - assert cfg.duration == 0.0 - assert cfg.max_missed_intervals == 5 - assert cfg.max_consecutive_errors == 3 - assert cfg.error_backoff_base == 1.0 - - def test_custom_values(self): - cfg = PollingConfig(interval=0.5, duration=10.0, max_missed_intervals=2) - assert cfg.interval == 0.5 - assert cfg.duration == 10.0 - assert cfg.max_missed_intervals == 2 - - def test_is_frozen(self): - cfg = PollingConfig() - with pytest.raises((AttributeError, TypeError)): - cfg.interval = 99.0 # type: ignore[misc] - - -class TestCursorInvalidatedError: - def test_is_exception(self): - e = CursorInvalidatedError("state lost") - assert isinstance(e, Exception) - assert "state lost" in str(e) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v 2>&1 | head -30 -``` - -Expected: `ImportError` or `FAILED` — `Cursor`, `PollingConfig`, `CursorInvalidatedError` not yet defined. - -- [ ] **Step 3: Add `T`, `Cursor[T]`, `PollingConfig` to `types.py`** - -At the top of `src/orcapod/types.py`, change the existing `typing` import line from: -```python -from typing import TYPE_CHECKING, Any, Self, TypeAlias -``` -to: -```python -from datetime import datetime -from typing import TYPE_CHECKING, Any, Generic, Self, TypeAlias, TypeVar -``` - -Then add the following block **after** the existing `SchemaLike` alias (around line 55, before the `Schema` class): - -```python -T = TypeVar("T") -"""Generic cursor value type for ``Cursor`` and ``DynamicSourceProtocol``.""" -``` - -Then add the following block **at the end of `types.py`**, after the `ColumnInfo` class: - -```python -@dataclass -class Cursor(Generic[T]): - """Marks the current position in a DynamicSource's data stream. - - Args: - value: Implementation-defined cursor value. May be a datetime - timestamp, integer offset, string pagination token, or any - other type meaningful to the implementation. - modified_at: Optional wall-clock time when the source content at - this cursor position was last modified. When provided, - ``PollingSource`` uses this to update its ``last_modified`` - timestamp for downstream staleness detection. When ``None``, - the framework falls back to its own wall clock. - """ - - value: T - modified_at: datetime | None = None - - -@dataclass(frozen=True) -class PollingConfig: - """Configuration for a ``PollingSource``. - - Args: - interval: Seconds between ``poll()`` calls, measured start-to-start. - duration: Total seconds to run. ``0`` means indefinite (run until - cancelled). - max_missed_intervals: Maximum consecutive tick windows consumed by a - single poll+fetch cycle before the source terminates. - Resets to zero on any clean tick. - max_consecutive_errors: Maximum consecutive ``poll()``/``fetch()`` - failures before the source closes its channel cleanly. - error_backoff_base: Base wait in seconds for exponential backoff on - errors. Wait after the nth error is - ``error_backoff_base * 2 ** (n - 1)``. - """ - - interval: float = 1.0 - duration: float = 0.0 - max_missed_intervals: int = 5 - max_consecutive_errors: int = 3 - error_backoff_base: float = 1.0 -``` - -- [ ] **Step 4: Add `CursorInvalidatedError` to `errors.py`** - -Append to `src/orcapod/errors.py`: - -```python -class CursorInvalidatedError(Exception): - """Raised by a ``DynamicSourceProtocol`` implementation when the previous - cursor is no longer valid and the source state must be rebuilt from scratch. - - This is a terminal condition for ``PollingSource``. Rows already emitted - downstream cannot be retracted, so continuing would leave downstream - operators with a corrupted view. ``PollingSource`` catches this, logs a - clear error, closes its output channel cleanly, and calls ``close()``. - - If full-reset semantics are required, use a static source re-run instead - of ``PollingSource``. - """ -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestCursor tests/test_channels/test_polling_source.py::TestPollingConfig tests/test_channels/test_polling_source.py::TestCursorInvalidatedError -v -``` - -Expected: all 9 tests `PASSED`. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/types.py src/orcapod/errors.py tests/test_channels/test_polling_source.py -git commit -m "feat(types): add Cursor[T], PollingConfig, CursorInvalidatedError for polling source" -``` - ---- - -## Task 2: `DynamicSourceProtocol[T]` - -**Files:** -- Modify: `src/orcapod/protocols/core_protocols/sources.py` -- Modify: `src/orcapod/protocols/core_protocols/__init__.py` -- Test: `tests/test_channels/test_polling_source.py` (append) - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/test_channels/test_polling_source.py`: - -```python -# =========================================================================== -# Task 2: DynamicSourceProtocol conformance -# =========================================================================== - - -from orcapod.protocols.core_protocols.sources import DynamicSourceProtocol - - -class _MinimalImpl: - """Minimal protocol-conformant implementation for isinstance checks.""" - - async def poll(self, cursor: Cursor[int] | None = None) -> bool: - return False - - async def fetch(self, cursor: Cursor[int] | None = None) -> tuple[Cursor[int], dict]: - return Cursor(value=0), {} - - async def close(self) -> None: - pass - - -class TestDynamicSourceProtocol: - def test_minimal_impl_satisfies_protocol(self): - impl = _MinimalImpl() - assert isinstance(impl, DynamicSourceProtocol) - - def test_missing_poll_fails_isinstance(self): - class NoPoll: - async def fetch(self, cursor=None): - return Cursor(value=0), {} - - async def close(self): - pass - - assert not isinstance(NoPoll(), DynamicSourceProtocol) - - def test_missing_fetch_fails_isinstance(self): - class NoFetch: - async def poll(self, cursor=None): - return False - - async def close(self): - pass - - assert not isinstance(NoFetch(), DynamicSourceProtocol) - - def test_missing_close_fails_isinstance(self): - class NoClose: - async def poll(self, cursor=None): - return False - - async def fetch(self, cursor=None): - return Cursor(value=0), {} - - assert not isinstance(NoClose(), DynamicSourceProtocol) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestDynamicSourceProtocol -v 2>&1 | head -20 -``` - -Expected: `ImportError` — `DynamicSourceProtocol` not yet defined. - -- [ ] **Step 3: Add `DynamicSourceProtocol[T]` to `sources.py`** - -Edit `src/orcapod/protocols/core_protocols/sources.py`. Replace the entire file with: - -```python -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Generic, Protocol, runtime_checkable - -from orcapod.protocols.core_protocols.streams import StreamProtocol -from orcapod.types import Cursor, T - -if TYPE_CHECKING: - from polars._typing import FrameInitTypes - - from orcapod.protocols.database_protocols import DatabaseRegistryProtocol - - -@runtime_checkable -class SourceProtocol(StreamProtocol, Protocol): - """ - Protocol for root sources — streams with no upstream dependencies that - expose provenance identity and optional field resolution. - - A SourceProtocol is a StreamProtocol where: - - ``source`` is always ``None`` (no upstream pod) - - ``upstreams`` is always empty - - ``source_id`` provides a canonical name for registry and provenance - - ``resolve_field`` enables lookup of individual field values by record id - """ - - @property - def source_id(self) -> str: ... - - def resolve_field(self, record_id: str, field_name: str) -> Any: ... - - def to_config( - self, db_registry: DatabaseRegistryProtocol | None = None - ) -> dict[str, Any]: - """Serialize source configuration to a JSON-compatible dict.""" - ... - - @classmethod - def from_config( - cls, - config: dict[str, Any], - db_registry: DatabaseRegistryProtocol | None = None, - ) -> "SourceProtocol": - """Reconstruct a source instance from a config dict.""" - ... - - -@runtime_checkable -class DynamicSourceProtocol(Protocol[T]): - """User-supplied protocol for a polling data source. - - Implementations provide three async methods. The framework handles - scheduling, cursor tracking, cache management, error handling, and - lifecycle. - - Type parameter ``T`` is the cursor value type (e.g. ``datetime``, ``int``, - ``str``). - - Cursor contract: - ``poll()`` returns ``True`` if new data is available since the given - cursor, ``False`` otherwise. Cursor advancement is tied to data - reading — ``fetch()`` returns a ``(new_cursor, data)`` tuple, and the - framework advances the cursor only after a successful fetch. - - Full-state invalidation: - Raise ``CursorInvalidatedError`` from ``poll()`` - or ``fetch()`` when previous state is no longer valid. This is a - terminal condition — ``PollingSource`` will close its channel cleanly. - - Example:: - - class MyDBSource: - def __init__(self, db): - self._db = db - - async def poll( - self, cursor: Cursor[datetime] | None = None - ) -> bool: - latest = await self._db.latest_modified_at() - return cursor is None or latest > cursor.value - - async def fetch( - self, cursor: Cursor[datetime] | None = None - ) -> tuple[Cursor[datetime], pl.DataFrame]: - since = cursor.value if cursor else None - df = await self._db.fetch_rows_since(since) - latest = await self._db.latest_modified_at() - return Cursor(value=latest, modified_at=latest), df - - async def close(self) -> None: - await self._db.disconnect() - """ - - async def poll(self, cursor: Cursor[T] | None = None) -> bool: - """Check whether new data is available. - - Args: - cursor: The framework's current cursor position, or ``None`` on - the first call. - - Returns: - ``True`` if new data is available since *cursor*, ``False`` if - nothing has changed. - - Raises: - CursorInvalidatedError: If previous state is no longer valid. - """ - ... - - async def fetch( - self, cursor: Cursor[T] | None = None - ) -> tuple[Cursor[T], "FrameInitTypes"]: - """Fetch data from the given cursor position onward. - - Called only when ``poll()`` has returned ``True``. Returns both the - new cursor position and the data so cursor advancement is always - tied to a successful read. - - Args: - cursor: The current cursor position, or ``None`` on the first - call. Implementations that cannot filter by cursor may ignore - this and return full state. - - Returns: - A tuple ``(new_cursor, data)`` where *data* is anything accepted - by ``pl.DataFrame()`` — polars DataFrame, pandas DataFrame, - PyArrow Table, dict, list, etc. - - Raises: - CursorInvalidatedError: If previous state is no longer valid. - """ - ... - - async def close(self) -> None: - """Release resources held by this source. - - Called on every termination path: normal duration expiry, pipeline - cancellation, max error threshold exceeded, or - ``CursorInvalidatedError``. The framework - guarantees ``close()`` is awaited before the output channel is closed. - """ - ... -``` - -- [ ] **Step 4: Export `DynamicSourceProtocol` from `core_protocols/__init__.py`** - -In `src/orcapod/protocols/core_protocols/__init__.py`, add the import and `__all__` entry: - -Change: -```python -from .sources import SourceProtocol -``` -to: -```python -from .sources import DynamicSourceProtocol, SourceProtocol -``` - -And in `__all__`, add `"DynamicSourceProtocol"` after `"SourceProtocol"`. - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestDynamicSourceProtocol -v -``` - -Expected: all 4 tests `PASSED`. - -- [ ] **Step 6: Run existing tests to catch regressions** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -10 -``` - -Expected: all existing tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/protocols/core_protocols/sources.py \ - src/orcapod/protocols/core_protocols/__init__.py \ - tests/test_channels/test_polling_source.py -git commit -m "feat(protocols): add DynamicSourceProtocol[T] for polling data sources" -``` - ---- - -## Task 3: `StreamBase.async_iter_data()` Default + `SourceNode.async_execute` Simplification - -**Files:** -- Modify: `src/orcapod/core/streams/base.py:242-253` -- Modify: `src/orcapod/core/nodes/source_node.py:300-326` -- Test: `tests/test_channels/test_polling_source.py` (append) - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/test_channels/test_polling_source.py`: - -```python -# =========================================================================== -# Task 3: StreamBase.async_iter_data default + SourceNode.async_execute -# =========================================================================== - - -from orcapod.channels import Channel -from orcapod.core.nodes.source_node import SourceNode -from orcapod.core.streams.arrow_table_stream import ArrowTableStream - - -def _make_arrow_stream(n: int = 3) -> ArrowTableStream: - """Build a minimal ArrowTableStream with `id` tag and `val` data column.""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("val", pa.int64(), nullable=False), - ]) - table = pa.table( - {"id": pa.array(list(range(n)), type=pa.int64()), - "val": pa.array([i * 10 for i in range(n)], type=pa.int64())}, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -class TestAsyncIterDataDefault: - @pytest.mark.asyncio - async def test_default_yields_same_items_as_iter_data(self): - stream = _make_arrow_stream(3) - sync_items = list(stream.iter_data()) - - async_items = [] - async for item in stream.async_iter_data(): - async_items.append(item) - - assert len(async_items) == 3 - assert len(async_items) == len(sync_items) - - @pytest.mark.asyncio - async def test_source_node_async_execute_uses_async_iter(self): - """SourceNode.async_execute routes through async_iter_data — no regression.""" - stream = _make_arrow_stream(2) - node = SourceNode(stream) - - ch = Channel(buffer_size=8) - await node.async_execute(ch.writer) - - rows = await ch.reader.collect() - assert len(rows) == 2 - - @pytest.mark.asyncio - async def test_source_node_closes_channel_after_exhaustion(self): - stream = _make_arrow_stream(1) - node = SourceNode(stream) - - ch = Channel(buffer_size=4) - await node.async_execute(ch.writer) - - # collect() only returns after channel is closed - rows = await ch.reader.collect() - assert len(rows) == 1 -``` - -- [ ] **Step 2: Run tests to verify they fail on the stub** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestAsyncIterDataDefault -v 2>&1 | head -20 -``` - -Expected: `FAILED` with `NotImplementedError` — the stub raises. - -- [ ] **Step 3: Replace the `async_iter_data` stub in `StreamBase`** - -In `src/orcapod/core/streams/base.py`, replace lines 242–253: - -```python - async def async_iter_data( - self, - ) -> AsyncIterator[tuple[TagProtocol, DataProtocol]]: - """Async iterator over (tag, data) pairs. - - Subclasses should override this to provide true async iteration. - """ - raise NotImplementedError( - f"{type(self).__name__} does not implement async_iter_data" - ) - # Make this an async generator so the return type is correct - yield # pragma: no cover -``` - -with: - -```python - async def async_iter_data( - self, - ) -> AsyncIterator[tuple[TagProtocol, DataProtocol]]: - """Async iterator over (tag, data) pairs. - - Default implementation wraps ``iter_data`` as an async generator. - Subclasses override this to provide true async streaming behaviour - (e.g. ``PollingSource``). - """ - for item in self.iter_data(): - yield item -``` - -- [ ] **Step 4: Simplify `SourceNode.async_execute`** - -In `src/orcapod/core/nodes/source_node.py`, replace the `async_execute` method (lines 300–326) with: - -```python - async def async_execute( - self, - output: WritableChannel[tuple[cp.TagProtocol, cp.DataProtocol]], - *, - observer: ExecutionObserverProtocol | None = None, - ) -> None: - """Push all (tag, data) pairs from the wrapped stream to the output channel. - - Delegates to ``async_iter_data`` - so that dynamic sources (e.g. ``PollingSource``) - stream continuously without modification to this node. - - Args: - output: Channel to write results to. - observer: Optional execution observer for hooks. - """ - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) - node_label = self.label - node_hash = "" - try: - if observer is not None: - observer.on_node_start(node_label, node_hash) - async for tag, data in self.stream.async_iter_data(): - await output.send((tag, data)) - if observer is not None: - observer.on_node_end(node_label, node_hash) - finally: - await output.close() -``` - -- [ ] **Step 5: Run the new tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestAsyncIterDataDefault -v -``` - -Expected: all 3 tests `PASSED`. - -- [ ] **Step 6: Run the full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -15 -``` - -Expected: all existing tests pass (the simplification is behaviourally equivalent). - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/streams/base.py \ - src/orcapod/core/nodes/source_node.py \ - tests/test_channels/test_polling_source.py -git commit -m "feat(streams): add async_iter_data default; simplify SourceNode.async_execute" -``` - ---- - -## Task 4: `PollingSource` — Sync Infrastructure - -**Files:** -- Create: `src/orcapod/core/sources/polling_source.py` -- Modify: `src/orcapod/core/sources/__init__.py` -- Test: `tests/test_channels/test_polling_source.py` (append) - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/test_channels/test_polling_source.py`: - -```python -# =========================================================================== -# Shared test fixture — FakeDynamicSource -# =========================================================================== - - -class FakeDynamicSource: - """Hand-written test double for DynamicSourceProtocol. - - Serves pre-loaded batches sequentially. ``poll()`` returns ``True`` - while un-fetched batches remain; ``False`` once all are consumed. - """ - - def __init__( - self, - batches: list[Any], - *, - poll_always_false: bool = False, - poll_raises: Exception | None = None, - fetch_raises: Exception | None = None, - fetch_delay: float = 0.0, - cursor_modified_at: datetime | None = None, - ) -> None: - self._batches = batches - self._batch_index = 0 - self._poll_always_false = poll_always_false - self._poll_raises = poll_raises - self._fetch_raises = fetch_raises - self._fetch_delay = fetch_delay - self._cursor_modified_at = cursor_modified_at - self.close_called = False - self.poll_cursors: list[Cursor[int] | None] = [] - self.fetch_cursors: list[Cursor[int] | None] = [] - - async def poll(self, cursor: Cursor[int] | None = None) -> bool: - self.poll_cursors.append(cursor) - if self._poll_raises is not None: - raise self._poll_raises - if self._poll_always_false: - return False - current_idx = cursor.value if cursor is not None else 0 - return current_idx < len(self._batches) - - async def fetch( - self, cursor: Cursor[int] | None = None - ) -> tuple[Cursor[int], Any]: - self.fetch_cursors.append(cursor) - if self._fetch_raises is not None: - raise self._fetch_raises - if self._fetch_delay > 0: - await asyncio.sleep(self._fetch_delay) - current_idx = cursor.value if cursor is not None else 0 - if current_idx >= len(self._batches): - return Cursor(value=current_idx, modified_at=self._cursor_modified_at), {} - data = self._batches[current_idx] - self._batch_index = current_idx + 1 - return ( - Cursor(value=current_idx + 1, modified_at=self._cursor_modified_at), - data, - ) - - async def close(self) -> None: - self.close_called = True - - -# =========================================================================== -# Task 4: PollingSource sync mode tests -# =========================================================================== - - -from orcapod.core.sources.polling_source import PollingSource - - -def _batch(id_: int, val: int) -> dict: - return {"id": pa.array([id_], type=pa.int64()), "val": pa.array([val], type=pa.int64())} - - -class TestPollingSourceSyncMode: - def test_sync_snapshot_fetches_on_first_iter_data(self): - """First iter_data() triggers fetch(cursor=None) and returns rows.""" - fake = FakeDynamicSource(batches=[_batch(1, 10)]) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - rows = list(src.iter_data()) - assert len(rows) == 1 - # First fetch always receives cursor=None - assert fake.fetch_cursors == [None] - - def test_sync_snapshot_fetch_called_once_on_repeated_access(self): - """Second iter_data() polls first; if poll returns False, cache is served.""" - fake = FakeDynamicSource(batches=[_batch(1, 10)], poll_always_false=True) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - list(src.iter_data()) # first call — fetches - list(src.iter_data()) # second call — polls (False) → serves cache - - # fetch called once (first access), poll called once (second access) - assert len(fake.fetch_cursors) == 1 - assert len(fake.poll_cursors) == 1 - - def test_sync_cache_miss_refetches_and_combines(self): - """Second iter_data() re-fetches when poll returns True.""" - fake = FakeDynamicSource( - batches=[_batch(1, 10), _batch(2, 20)] - ) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - rows1 = list(src.iter_data()) # fetches batch 0 - rows2 = list(src.iter_data()) # polls → True, fetches batch 1 → combines - - assert len(rows1) == 1 - assert len(rows2) == 2 # accumulated: batch 0 + batch 1 - - def test_cursor_threading_fetch_receives_previous_fetch_cursor(self): - """fetch() receives the cursor returned by the previous fetch call.""" - fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - list(src.iter_data()) # first fetch → cursor returned is Cursor(value=1) - list(src.iter_data()) # poll(Cursor(1)), then fetch(Cursor(1)) - - # poll should receive the cursor from the first fetch - assert fake.poll_cursors[0] is not None - assert fake.poll_cursors[0].value == 1 - # second fetch should receive same cursor - assert fake.fetch_cursors[1] is not None - assert fake.fetch_cursors[1].value == 1 - - def test_last_modified_updated_from_cursor_modified_at(self): - """cursor.modified_at updates last_modified; None falls back to wall clock.""" - ts = datetime(2026, 3, 15, 12, 0, 0, tzinfo=timezone.utc) - fake = FakeDynamicSource(batches=[_batch(1, 10)], cursor_modified_at=ts) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - list(src.iter_data()) - assert src.last_modified == ts - - def test_last_modified_falls_back_to_wall_clock_when_none(self): - """When cursor.modified_at is None, last_modified is updated to wall clock.""" - fake = FakeDynamicSource(batches=[_batch(1, 10)], cursor_modified_at=None) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - before = datetime.now(timezone.utc) - list(src.iter_data()) - after = datetime.now(timezone.utc) - - assert src.last_modified is not None - assert before <= src.last_modified <= after - - def test_output_schema_triggers_fetch(self): - """output_schema() lazily triggers fetch on first access.""" - fake = FakeDynamicSource(batches=[_batch(1, 10)]) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - tag_schema, data_schema = src.output_schema() - assert "id" in tag_schema - assert "val" in data_schema - - def test_keys_triggers_fetch(self): - fake = FakeDynamicSource(batches=[_batch(1, 10)]) - src = PollingSource(fake, tag_columns="id", polling_config=PollingConfig(interval=1.0)) - - tag_keys, data_keys = src.keys() - assert "id" in tag_keys - assert "val" in data_keys - - def test_identity_structure_schema_independent(self): - """identity_structure() does not require a fetch (schema-independent).""" - fake = FakeDynamicSource(batches=[]) - src = PollingSource( - fake, tag_columns="id", polling_config=PollingConfig(interval=1.0), - source_id="test_source" - ) - # Should not raise even though no fetch has occurred - ident = src.identity_structure() - assert "test_source" in str(ident) - - def test_to_config_returns_non_reconstructable_descriptor(self): - fake = FakeDynamicSource(batches=[]) - src = PollingSource( - fake, tag_columns="id", polling_config=PollingConfig(), - source_id="my_source" - ) - cfg = src.to_config() - assert cfg["source_type"] == "polling_source" - assert cfg["source_id"] == "my_source" - - def test_from_config_raises(self): - with pytest.raises(NotImplementedError): - PollingSource.from_config({"source_type": "polling_source"}) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSyncMode -v 2>&1 | head -20 -``` - -Expected: `ImportError` — `PollingSource` not yet defined. - -- [ ] **Step 3: Create `src/orcapod/core/sources/polling_source.py`** - -```python -"""Protocol-based polling source for async pipelines. - -Provides ``PollingSource``, a ``RootSource`` -that wraps a ``DynamicSourceProtocol`` -implementation. The framework handles scheduling, cursor tracking, cache management, -error handling, and shutdown; the implementation only supplies ``poll``, ``fetch``, -and ``close``. -""" -from __future__ import annotations - -import asyncio -import logging -from collections.abc import Collection -from math import floor -from typing import TYPE_CHECKING, Any, Generic - -from orcapod.core.sources.base import RootSource -from orcapod.core.sources.stream_builder import SourceStreamBuilder -from orcapod.errors import CursorInvalidatedError -from orcapod.types import ColumnConfig, Cursor, PollingConfig, T -from orcapod.utils import arrow_utils, polars_data_utils -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from polars._typing import FrameInitTypes - - from orcapod.core.streams.arrow_table_stream import ArrowTableStream - from orcapod.protocols.core_protocols.sources import DynamicSourceProtocol - from orcapod.types import Schema -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Module-level sync executor (mirrors data_function.py pattern) -# --------------------------------------------------------------------------- - -_sync_executor = None - - -def _get_sync_executor(): - global _sync_executor - if _sync_executor is None: - from concurrent.futures import ThreadPoolExecutor - - _sync_executor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="polling_source_sync" - ) - return _sync_executor - - -def _run_sync(coro): - """Run *coro* synchronously, safe even when called from within a running loop.""" - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(coro) - else: - return _get_sync_executor().submit(lambda: asyncio.run(coro)).result() - - -# --------------------------------------------------------------------------- -# PollingSource -# --------------------------------------------------------------------------- - - -class PollingSource(RootSource, Generic[T]): - """A root source that continuously emits data via a polling loop. - - Wraps a ``DynamicSourceProtocol`` - implementation. Under async execution (``async_iter_data``), the framework - polls on a fixed interval and yields new rows as they arrive. Under sync - execution (``iter_data``), a single poll+fetch cycle is performed on each - access and results are served from an accumulated in-memory cache. - - Args: - impl: User-supplied protocol implementation. - tag_columns: Column name(s) that form the tag (key) for each row. - All other columns are data columns. - polling_config: Scheduling and error configuration. - source_id: Optional stable identifier for this source. - label: Optional human-readable label. - data_context: Optional data context key or instance. - config: Optional Orcapod framework config. - - Note: - Sync mode calls ``asyncio.run()`` (or a thread executor when inside a - running loop). This fails without a shim in Jupyter notebooks. Async - mode has no such restriction. - - Note: - The accumulated cache is unbounded for true-delta implementations. - No eviction policy is implemented in this version. - """ - - def __init__( - self, - impl: DynamicSourceProtocol[T], - tag_columns: str | Collection[str], - polling_config: PollingConfig = PollingConfig(), - source_id: str | None = None, - label: str | None = None, - data_context: str | Any | None = None, - config: Any | None = None, - ) -> None: - super().__init__( - source_id=source_id, - label=label, - data_context=data_context, - config=config, - ) - self._impl: DynamicSourceProtocol[T] = impl - if isinstance(tag_columns, str): - self._tag_columns: tuple[str, ...] = (tag_columns,) - else: - self._tag_columns = tuple(tag_columns) - self._polling_config = polling_config - self._cursor: Cursor[T] | None = None - self._schema_stream: ArrowTableStream | None = None - - # ------------------------------------------------------------------------- - # Identity - # ------------------------------------------------------------------------- - - def identity_structure(self) -> Any: - """Schema-independent identity (schema is unknown until first fetch).""" - return (self.__class__.__name__, self._tag_columns, self._source_id or "") - - # ------------------------------------------------------------------------- - # Sync stream delegation — all route through _get_latest_stream() - # ------------------------------------------------------------------------- - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - return self._get_latest_stream().output_schema(columns=columns, all_info=all_info) - - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - return self._get_latest_stream().keys(columns=columns, all_info=all_info) - - def iter_data(self): - return self._get_latest_stream().iter_data() - - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table: - return self._get_latest_stream().as_table(columns=columns, all_info=all_info) - - # ------------------------------------------------------------------------- - # Serialization - # ------------------------------------------------------------------------- - - def to_config(self, db_registry: Any = None) -> dict[str, Any]: - """Serialize this source to a JSON-compatible dict. - - Delegates to the ``DynamicSourceProtocol`` implementation's - ``to_config()``. If the implementation returns ``None`` (not - serializable), returns a non-reconstructable descriptor. - """ - impl_config = self._impl.to_config() - if impl_config is None: - return { - "source_type": "polling_source", - "reconstructable": False, - "source_id": self._source_id, - } - return { - "source_type": "polling_source", - "impl_class": f"{type(self._impl).__module__}.{type(self._impl).__qualname__}", - "impl_config": impl_config, - "tag_columns": list(self._tag_columns), - "source_id": self._source_id, - } - - @classmethod - def from_config(cls, config: dict[str, Any], db_registry: Any = None) -> PollingSource: - """Reconstruct a ``PollingSource`` from a config dict. - - Uses the stored ``impl_class`` to import and call - ``DynamicSourceProtocol.from_config(impl_config)``. - - Raises: - NotImplementedError: If the stored config has ``reconstructable: False``. - """ - if not config.get("reconstructable", True): - raise NotImplementedError( - "PollingSource cannot be reconstructed — " - "the DynamicSourceProtocol implementation is not serializable." - ) - import importlib - - module_name, _, class_name = config["impl_class"].rpartition(".") - module = importlib.import_module(module_name) - impl_cls = getattr(module, class_name) - impl = impl_cls.from_config(config["impl_config"]) - return cls( - impl, - tag_columns=config["tag_columns"], - source_id=config.get("source_id"), - ) - - # ------------------------------------------------------------------------- - # Internal sync helpers - # ------------------------------------------------------------------------- - - def _get_latest_stream(self) -> ArrowTableStream: - """Return the current accumulated stream, fetching/polling as needed.""" - if self._schema_stream is None: - # First access — no cache yet; fetch immediately - logger.debug("PollingSource %r: first sync access — fetching", self._source_id) - new_cursor, data = _run_sync(self._impl.fetch(cursor=None)) - self._schema_stream = self._build_stream(data) - self._cursor = new_cursor - self._update_last_modified_from_cursor(new_cursor) - else: - # Have cache — poll for updates - has_new = _run_sync(self._impl.poll(cursor=self._cursor)) - if has_new: - logger.debug("PollingSource %r: sync poll found new data — fetching", self._source_id) - new_cursor, data = _run_sync(self._impl.fetch(cursor=self._cursor)) - self._schema_stream = self._combine(self._schema_stream, data) - self._cursor = new_cursor - self._update_last_modified_from_cursor(new_cursor) - else: - logger.debug("PollingSource %r: sync poll — cache still valid", self._source_id) - return self._schema_stream - - def _build_stream(self, data: FrameInitTypes) -> ArrowTableStream: - """Build an ``ArrowTableStream`` from raw ``FrameInitTypes`` data.""" - from orcapod.core.streams.arrow_table_stream import ArrowTableStream - - df = pl.DataFrame(data) - - # Handle Object-dtype columns (same pattern as DataFrameSource) - object_columns = [c for c in df.columns if df[c].dtype == pl.Object] - if object_columns: - sub_table = self.data_context.type_converter.python_dicts_to_arrow_table( - df.select(object_columns).to_dicts() - ) - df = df.with_columns([pl.from_arrow(c) for c in sub_table]) - - df = polars_data_utils.drop_system_columns(df) - - arrow_table = df.to_arrow() - arrow_table = arrow_table.cast(arrow_utils.infer_schema_nullable(arrow_table)) - - builder = SourceStreamBuilder(self.data_context, self.orcapod_config) - result = builder.build( - arrow_table, - tag_columns=self._tag_columns, - source_id=self._source_id, - ) - - if self._source_id is None: - self._source_id = result.source_id - - return result.stream - - def _combine( - self, existing: ArrowTableStream, new_stream: ArrowTableStream - ) -> ArrowTableStream: - """Append *new_stream* rows to *existing*, raising on schema mismatch. - - Raises: - InputValidationError: If the column sets differ or any column type - has changed between batches. - """ - from orcapod.core.streams.arrow_table_stream import ArrowTableStream - - self._validate_combining_schemas(existing, new_stream) - - combined = pa.concat_tables( - [ - existing.as_table(all_info=True), - new_stream.as_table(all_info=True), - ], - ) - return ArrowTableStream(table=combined, tag_columns=existing._tag_columns) - - def _update_last_modified_from_cursor(self, cursor: Cursor[T]) -> None: - """Update ``last_modified`` from cursor or fall back to wall clock.""" - if cursor.modified_at is not None: - self._set_modified_time(cursor.modified_at) - else: - self._update_modified_time() - - # ------------------------------------------------------------------------- - # Async mode — full polling loop - # ------------------------------------------------------------------------- - - async def async_iter_data(self): - """Async generator that continuously emits (tag, data) pairs. - - Pre-seeds from the cached stream (if any) before entering the polling - loop. The loop runs until: the configured duration elapses, the maximum - consecutive error or overrun threshold is exceeded, a - ``CursorInvalidatedError`` is raised, or the task - is cancelled. - - ``impl.close()`` is always awaited before returning. - """ - # Pre-seed from cache - if self._schema_stream is not None: - logger.debug("PollingSource %r: pre-seeding %d cached row(s)", self._source_id, len(list(self._schema_stream.iter_data()))) - for item in self._schema_stream.iter_data(): - yield item - - cfg = self._polling_config - loop = asyncio.get_running_loop() - start_time = loop.time() - next_tick = start_time - consecutive_misses = 0 - consecutive_errors = 0 - - logger.info( - "PollingSource %r starting (interval=%.2fs, duration=%.1fs)", - self._source_id, - cfg.interval, - cfg.duration, - ) - - try: - while True: - # 1. Sleep until next scheduled tick - now = loop.time() - if next_tick > now: - await asyncio.sleep(next_tick - now) - - # 2. Poll + fetch - try: - has_new = await self._impl.poll(cursor=self._cursor) - - if has_new: - logger.debug( - "PollingSource %r: new data detected, fetching", - self._source_id, - ) - new_cursor, data = await self._impl.fetch(cursor=self._cursor) - if self._schema_stream is None: - self._schema_stream = self._build_stream(data) - else: - self._schema_stream = self._combine(self._schema_stream, data) - self._cursor = new_cursor - self._update_last_modified_from_cursor(new_cursor) - - # Collect and emit new rows - new_stream = self._build_stream(data) - rows = list(new_stream.iter_data()) - logger.debug( - "PollingSource %r: emitting %d row(s)", - self._source_id, - len(rows), - ) - for item in rows: - yield item - else: - logger.debug( - "PollingSource %r: poll returned no new data", - self._source_id, - ) - - consecutive_errors = 0 - - except asyncio.CancelledError: - raise - - except CursorInvalidatedError: - logger.error( - "PollingSource %r: cursor invalidated — previous state cannot " - "be reconciled with already-emitted rows. Terminating source.", - self._source_id, - ) - return - - except Exception as e: - consecutive_errors += 1 - backoff = cfg.error_backoff_base * 2 ** (consecutive_errors - 1) - logger.error( - "PollingSource %r: poll/fetch error (consecutive=%d, " - "backoff=%.1fs): %s", - self._source_id, - consecutive_errors, - backoff, - e, - ) - if consecutive_errors >= cfg.max_consecutive_errors: - logger.error( - "PollingSource %r: max consecutive errors (%d) reached. " - "Terminating source.", - self._source_id, - cfg.max_consecutive_errors, - ) - return - await asyncio.sleep(backoff) - continue # retry — do not advance next_tick - - # 3. Tick advancement (start-to-start) - now = loop.time() - intervals_consumed = floor((now - next_tick) / cfg.interval) - if intervals_consumed > 0: - consecutive_misses += intervals_consumed - logger.warning( - "PollingSource %r: tick overrun — consumed %d interval(s) " - "(consecutive_misses=%d/%d)", - self._source_id, - intervals_consumed, - consecutive_misses, - cfg.max_missed_intervals, - ) - if consecutive_misses >= cfg.max_missed_intervals: - logger.error( - "PollingSource %r: overrun threshold exceeded. " - "Terminating source.", - self._source_id, - ) - return - else: - consecutive_misses = 0 - next_tick += (intervals_consumed + 1) * cfg.interval - - # 4. Duration check - if cfg.duration > 0 and (loop.time() - start_time) >= cfg.duration: - logger.info( - "PollingSource %r: duration limit (%.1fs) reached. " - "Terminating source.", - self._source_id, - cfg.duration, - ) - return - - except asyncio.CancelledError: - logger.info( - "PollingSource %r: cancelled — shutting down cleanly.", - self._source_id, - ) - - finally: - logger.debug("PollingSource %r: calling impl.close()", self._source_id) - await self._impl.close() - logger.info("PollingSource %r: closed.", self._source_id) -``` - -- [ ] **Step 4: Export `PollingSource` from `sources/__init__.py`** - -In `src/orcapod/core/sources/__init__.py`, add: - -```python -from .polling_source import PollingSource -``` - -And add `"PollingSource"` to `__all__`. - -- [ ] **Step 5: Run sync tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSyncMode -v -``` - -Expected: all 10 tests `PASSED`. - -- [ ] **Step 6: Run full suite** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -10 -``` - -Expected: no regressions. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/sources/polling_source.py \ - src/orcapod/core/sources/__init__.py \ - tests/test_channels/test_polling_source.py -git commit -m "feat(sources): add PollingSource sync infrastructure" -``` - ---- - -## Task 5: `PollingSource` — Async Polling Loop Tests (Basic) - -**Files:** -- Test: `tests/test_channels/test_polling_source.py` (append) - -The implementation was already added in Task 4's `polling_source.py`. This task adds and verifies the async tests. - -- [ ] **Step 1: Write the async tests** - -Append to `tests/test_channels/test_polling_source.py`: - -```python -# =========================================================================== -# Task 5: PollingSource async mode tests (basic) -# =========================================================================== - - -class TestPollingSourceAsyncMode: - @pytest.mark.asyncio - async def test_async_streaming_emits_rows(self): - """Positive polls trigger fetch; rows are emitted to async_iter_data.""" - fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.5), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert len(items) == 2 - assert fake.close_called - - @pytest.mark.asyncio - async def test_negative_poll_emits_nothing(self): - """poll() returning False emits nothing; fetch is never called.""" - fake = FakeDynamicSource(batches=[_batch(1, 10)], poll_always_false=True) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.05), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert len(items) == 0 - assert len(fake.fetch_cursors) == 0 - - @pytest.mark.asyncio - async def test_pre_seeding_yields_cached_rows_first(self): - """async_iter_data() yields cached rows before entering the polling loop.""" - fake = FakeDynamicSource(batches=[_batch(1, 10)]) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=1.0), - ) - - # Seed the cache synchronously first - list(src.iter_data()) - - # Now supply a second fake with no new data - fake2 = FakeDynamicSource(batches=[], poll_always_false=True) - src._impl = fake2 - - # async_iter_data should yield the 1 cached row, then stop after duration - items = [] - src._polling_config = PollingConfig(interval=0.01, duration=0.05) - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert len(items) == 1 # pre-seeded row - assert fake2.close_called - - @pytest.mark.asyncio - async def test_cache_combining_accumulates_rows(self): - """After two delta fetches, async stream contains rows from both batches.""" - fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.3), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert len(items) == 2 - # Internal cache should hold both rows - assert src._schema_stream is not None - cached_rows = list(src._schema_stream.iter_data()) - assert len(cached_rows) == 2 - - @pytest.mark.asyncio - async def test_clean_shutdown_on_cancellation(self): - """CancelledError causes close() to be awaited before terminating.""" - fake = FakeDynamicSource(batches=[], poll_always_false=True) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.0), - ) - - async def run(): - async for _ in src.async_iter_data(): - pass - - task = asyncio.create_task(run()) - await asyncio.sleep(0.05) - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - assert fake.close_called - - @pytest.mark.asyncio - async def test_duration_limit_terminates_source(self): - """Source stops naturally after config.duration seconds.""" - fake = FakeDynamicSource(batches=[], poll_always_false=True) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.08), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert fake.close_called - - @pytest.mark.asyncio - async def test_indefinite_mode_runs_until_cancelled(self): - """duration=0 runs until explicitly cancelled.""" - fake = FakeDynamicSource(batches=[], poll_always_false=True) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.0), - ) - - async def run(): - async for _ in src.async_iter_data(): - pass - - task = asyncio.create_task(run()) - # Task should still be running after a short sleep - await asyncio.sleep(0.05) - assert not task.done() - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - assert fake.close_called - - @pytest.mark.asyncio - async def test_cursor_threaded_through_async_fetches(self): - """In async mode fetch() receives the cursor from the previous fetch.""" - fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.3), - ) - - async for _ in src.async_iter_data(): - pass - - # First fetch: cursor=None; second fetch: cursor=Cursor(value=1) - assert fake.fetch_cursors[0] is None - if len(fake.fetch_cursors) > 1: - assert fake.fetch_cursors[1] is not None - assert fake.fetch_cursors[1].value == 1 -``` - -- [ ] **Step 2: Run the async tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceAsyncMode -v -``` - -Expected: all 8 tests `PASSED`. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_channels/test_polling_source.py -git commit -m "test(sources): add async polling loop tests for PollingSource" -``` - ---- - -## Task 6: `PollingSource` — Error Handling, Overrun, and Schema Drift - -**Files:** -- Test: `tests/test_channels/test_polling_source.py` (append) - -- [ ] **Step 1: Write the tests** - -Append to `tests/test_channels/test_polling_source.py`: - -```python -# =========================================================================== -# Task 6: Error handling, overrun, schema drift tests -# =========================================================================== - - -class TestPollingSourceErrorHandling: - @pytest.mark.asyncio - async def test_error_backoff_retries_and_uses_exponential_wait(self): - """Exception in poll() increments error counter; errors below threshold → retry.""" - - class CountingError(Exception): - pass - - call_count = 0 - - class TransientImpl: - async def poll(self, cursor=None) -> bool: - nonlocal call_count - call_count += 1 - if call_count <= 2: - raise CountingError("transient") - return False # succeed on 3rd attempt - - async def fetch(self, cursor=None): - return Cursor(value=0), {} - - async def close(self): - pass - - src = PollingSource( - TransientImpl(), - tag_columns="id", - polling_config=PollingConfig( - interval=0.01, - duration=0.5, - max_consecutive_errors=5, - error_backoff_base=0.01, # very short backoff for tests - ), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - # Completed without raising; errors were below threshold - assert call_count >= 3 - - @pytest.mark.asyncio - async def test_max_consecutive_errors_closes_cleanly(self): - """Reaching max_consecutive_errors closes the channel without crashing.""" - fake = FakeDynamicSource( - batches=[], - poll_raises=RuntimeError("persistent error"), - ) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig( - interval=0.01, - max_consecutive_errors=3, - error_backoff_base=0.01, - ), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert len(items) == 0 - assert fake.close_called - - @pytest.mark.asyncio - async def test_cursor_invalidated_error_terminates_without_retry(self): - """CursorInvalidatedError terminates the source cleanly; no retry.""" - fake = FakeDynamicSource( - batches=[], - poll_raises=CursorInvalidatedError("state lost"), - ) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.01), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - # Only one poll attempt — no retry for invalidation - assert len(fake.poll_cursors) == 1 - assert fake.close_called - - @pytest.mark.asyncio - async def test_overrun_terminates_after_threshold(self): - """Slow fetches that consistently overrun max_missed_intervals terminate the source.""" - fake = FakeDynamicSource( - batches=[_batch(i, i * 10) for i in range(20)], - fetch_delay=0.05, # each fetch takes 50 ms - ) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig( - interval=0.01, # 10 ms tick — fetch will always overrun - max_missed_intervals=3, # terminate after 3 cumulative misses - duration=0.0, # run indefinitely (until overrun) - ), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - # Terminated due to overrun; close() must have been called - assert fake.close_called - - @pytest.mark.asyncio - async def test_schema_drift_emits_warning(self, caplog): - """Mismatched columns on second fetch emit a WARNING log.""" - - class DriftingImpl: - _call = 0 - - async def poll(self, cursor=None) -> bool: - return self._call < 2 - - async def fetch(self, cursor=None): - self._call += 1 - if self._call == 1: - return ( - Cursor(value=1), - {"id": pa.array([1], type=pa.int64()), - "val": pa.array([10], type=pa.int64())}, - ) - # Second fetch has an extra column - return ( - Cursor(value=2), - {"id": pa.array([2], type=pa.int64()), - "val": pa.array([20], type=pa.int64()), - "extra": pa.array(["x"], type=pa.large_string())}, - ) - - async def close(self): - pass - - src = PollingSource( - DriftingImpl(), - tag_columns="id", - polling_config=PollingConfig(interval=0.01, duration=0.3), - ) - - with caplog.at_level(logging.WARNING, logger="orcapod.core.sources.polling_source"): - async for _ in src.async_iter_data(): - pass - - assert any("schema drift" in r.message for r in caplog.records) -``` - -- [ ] **Step 2: Run the tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceErrorHandling -v -``` - -Expected: all 5 tests `PASSED`. - -- [ ] **Step 3: Run the full suite** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -10 -``` - -Expected: no regressions. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_channels/test_polling_source.py -git commit -m "test(sources): add error handling, overrun, and schema drift tests for PollingSource" -``` - ---- - -## Task 7: Full Test Run + Cleanup - -**Files:** -- Review/clean: `tests/test_channels/test_polling_source.py` - -- [ ] **Step 1: Run the complete polling source test file** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v 2>&1 | tail -40 -``` - -Expected: all tests `PASSED`. Count of tests should be at least 19. - -- [ ] **Step 2: Run the complete test suite** - -```bash -uv run pytest tests/ -q 2>&1 | tail -15 -``` - -Expected: all tests pass with zero failures. - -- [ ] **Step 3: Verify exports are importable from public API** - -```bash -uv run python -c " -from orcapod.core.sources import PollingSource -from orcapod.protocols.core_protocols import DynamicSourceProtocol -from orcapod.types import Cursor, PollingConfig -from orcapod.errors import CursorInvalidatedError -print('All public exports OK') -print(f' PollingSource: {PollingSource}') -print(f' DynamicSourceProtocol: {DynamicSourceProtocol}') -print(f' Cursor: {Cursor}') -print(f' PollingConfig: {PollingConfig}') -print(f' CursorInvalidatedError: {CursorInvalidatedError}') -" -``` - -Expected output: -``` -All public exports OK - PollingSource: - DynamicSourceProtocol: - Cursor: - PollingConfig: - CursorInvalidatedError: -``` - -- [ ] **Step 4: Final commit** - -```bash -git add . -git commit -m "chore(sources): verify all polling source exports accessible" -``` - ---- - -## Notes for Implementors - -**`_run_sync` pattern:** Mirrors `data_function.py`. When `asyncio.get_running_loop()` raises `RuntimeError`, there is no running loop and `asyncio.run()` is safe. When already inside a loop (e.g. in a test), offload to a `ThreadPoolExecutor` thread with its own loop. - -**Pre-seeding in `async_iter_data`:** The pre-seed iterates `self._schema_stream.iter_data()` before yielding from it, only to get the count for the log message. This double-iterates. A simpler approach is to yield directly and log the count after: -```python -pre_seed_count = 0 -for item in self._schema_stream.iter_data(): - yield item - pre_seed_count += 1 -logger.debug("PollingSource %r: pre-seeded %d row(s)", self._source_id, pre_seed_count) -``` - -**Async emit vs. cache update:** In `async_iter_data`, new rows are converted to stream items by calling `_build_stream(data).iter_data()`. This is slightly redundant with `_combine()` which also calls `_build_stream()`. A future optimization could share the built stream, but correctness takes priority. - -**`pa.concat_tables` schema compatibility:** When combining two streams built from the same `tag_columns` and `source_id`, their schemas should be compatible. If schema drift occurred and a column was added, pass `promote_options="default"` to `pa.concat_tables` to fill missing fields with nulls. - -**Branch:** `eywalker/plt-1430-dynamic-data-source-protocol-based-polling-source-for-async` diff --git a/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md b/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md deleted file mode 100644 index 635074e6f..000000000 --- a/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md +++ /dev/null @@ -1,2995 +0,0 @@ -# Pipeline Pure-Descriptor Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `Pipeline` a pure computational descriptor (lightweight nodes only, no DB) while `PipelineJob` becomes the sole stateful, executable form — with `FunctionJobNode`/`OperatorJobNode`/`SourceJobNode` carrying all execution state. - -**Architecture:** Create parallel "blueprint" and "execution" node variants sharing base classes. `SourceNode` (schema-only) replaces `SourceSpec` with bit-identical hashes. `FunctionNode`/`OperatorNode` become thin wrappers (no DB); `FunctionJobNode`/`OperatorJobNode` carry all DB logic. `AbstractPipelineBase` extracts shared recording machinery. `Pipeline.bind()` is removed; `PipelineJob.from_pipeline()` is its replacement. `PipelineJob.bind()` becomes mutating (returns `None`). `SourceSpec` is deleted. - -**Tech Stack:** Python, PyArrow, NetworkX; always run `uv run pytest` (never `pytest` directly) - ---- - -## File Layout - -**New files:** -- `src/orcapod/pipeline/base.py` — `AbstractPipelineBase` (shared recording mechanism) -- `tests/test_core/nodes/test_source_node.py` — New unit tests for `SourceNode`/`SourceJobNode` -- `tests/test_core/nodes/test_function_node_split.py` — New unit tests for split hierarchy -- `tests/test_core/nodes/test_operator_node_split.py` — New unit tests for split hierarchy - -**Heavily modified (in execution order):** -1. `src/orcapod/errors.py` — Add `PipelineJobRequiredError` -2. `src/orcapod/core/nodes/source_node.py` — Full rewrite: `SourceNodeBase` + `SourceNode` (schema-only) + `SourceJobNode` -3. `src/orcapod/core/nodes/function_node.py` — Split: `FunctionNodeBase` + thin `FunctionNode` + `FunctionJobNode` -4. `src/orcapod/core/nodes/operator_node.py` — Split: `OperatorNodeBase` + thin `OperatorNode` + `OperatorJobNode` -5. `src/orcapod/core/nodes/__init__.py` — Add `FunctionJobNode`, `OperatorJobNode`, `SourceJobNode` exports; add `JobNode` alias -6. `src/orcapod/pipeline/base.py` ← NEW -7. `src/orcapod/pipeline/graph.py` — `Pipeline`: accept `SourceNode` leaves (not `SourceSpec`), use thin nodes, remove `bind()` -8. `src/orcapod/pipeline/job.py` — `PipelineJob`: use `JobNode` types, `from_pipeline()`, mutating `bind()`, `as_pipeline()` -9. `src/orcapod/pipeline/serialization.py` — New source serialization format, version bumps -10. `src/orcapod/__init__.py` — Remove `SourceSpec`, export `SourceNode` - -**Deleted:** -- `src/orcapod/core/sources/source_spec.py` -- `tests/test_core/sources/test_source_spec.py` - ---- - -## Critical Hash-Stability Invariant - -`SourceNode.identity_structure()` **must** return `("SourceSpec", name, tag_schema, data_schema)` — identical to the old `SourceSpec.identity_structure()`. This ensures: - -``` -SourceNode("x", tag_s, data_s).content_hash() == SourceSpec("x", tag_s, data_s).content_hash() -``` - -DB paths derived from `pipeline_hash()` chains must not change. Every task that touches identity structures must re-verify this invariant with tests before committing. - ---- - -## Task 1: Add PipelineJobRequiredError and rewrite source_node.py - -**Files:** -- Modify: `src/orcapod/errors.py` -- Rewrite: `src/orcapod/core/nodes/source_node.py` -- Create: `tests/test_core/nodes/test_source_node.py` - -- [ ] **Step 1.1: Add PipelineJobRequiredError to errors.py** - -Open `src/orcapod/errors.py` and append after the existing `SourceSpecMismatchError` class: - -```python -class PipelineJobRequiredError(RuntimeError): - """Raised when a lightweight blueprint node is asked to produce data. - - Blueprint nodes (``FunctionNode``, ``OperatorNode``) carry no database - references. Wrap the containing ``Pipeline`` in a ``PipelineJob`` to - obtain executable ``FunctionJobNode`` / ``OperatorJobNode`` variants. - """ -``` - -- [ ] **Step 1.2: Write failing tests for new SourceNode/SourceJobNode interface** - -Create `tests/test_core/nodes/` directory if it doesn't exist, and create -`tests/test_core/nodes/__init__.py` (empty) and -`tests/test_core/nodes/test_source_node.py`: - -```python -"""Tests for SourceNode (schema-only slot) and SourceJobNode (execution variant).""" -from __future__ import annotations - -import pytest - -from orcapod.errors import SourceSpecMismatchError, UnboundSourceError -from orcapod.types import Schema - - -@pytest.fixture -def tag_schema(): - return Schema({"id": int}) - - -@pytest.fixture -def data_schema(): - return Schema({"value": float}) - - -class TestSourceNodeHashStability: - """SourceNode must produce bit-identical hashes to SourceSpec with the same args.""" - - def test_content_hash_matches_source_spec(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.source_spec import SourceSpec - - spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - assert node.content_hash() == spec.content_hash() - - def test_pipeline_hash_matches_source_spec(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.source_spec import SourceSpec - - spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - assert node.pipeline_hash() == spec.pipeline_hash() - - def test_different_names_different_content_hash(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - - a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - b = SourceNode(name="slot_b", tag_schema=tag_schema, data_schema=data_schema) - assert a.content_hash() != b.content_hash() - - def test_different_names_same_pipeline_hash(self, tag_schema, data_schema): - """pipeline_hash is schema-only, name-independent.""" - from orcapod.core.nodes.source_node import SourceNode - - a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - b = SourceNode(name="slot_b", tag_schema=tag_schema, data_schema=data_schema) - assert a.pipeline_hash() == b.pipeline_hash() - - -class TestSourceNodeInterface: - def test_iter_data_raises_unbound_error(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - with pytest.raises(UnboundSourceError): - list(node.iter_data()) - - def test_output_schema(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - t, d = node.output_schema() - assert t == tag_schema - assert d == data_schema - - def test_label_resolves_to_name(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="my_slot", tag_schema=tag_schema, data_schema=data_schema) - assert node.label == "my_slot" - - def test_node_type(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - assert node.node_type == "source" - - def test_name_property(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="my_slot", tag_schema=tag_schema, data_schema=data_schema) - assert node.name == "my_slot" - - def test_validate_compatible_source(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.dict_source import DictSource - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - node.validate(src) # must not raise - - def test_validate_incompatible_raises(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.dict_source import DictSource - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(records=[{"id": 1, "wrong": 1.0}], tag_columns=["id"]) - with pytest.raises(SourceSpecMismatchError): - node.validate(src) - - -class TestSourceJobNode: - def test_unbound_iter_data_raises(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceJobNode - - job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - with pytest.raises(UnboundSourceError): - list(job_node.iter_data()) - - def test_unbound_content_hash_matches_source_node(self, tag_schema, data_schema): - """Unbound SourceJobNode has same content_hash as SourceNode.""" - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - assert job_node.content_hash() == node.content_hash() - - def test_pipeline_hash_matches_source_node(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - assert job_node.pipeline_hash() == node.pipeline_hash() - - def test_bound_content_hash_is_concrete_hash(self, tag_schema, data_schema): - """Bound SourceJobNode content_hash() == concrete.content_hash().""" - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - from orcapod.core.sources.dict_source import DictSource - - src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src - ) - assert job_node.content_hash() == src.content_hash() - - def test_bound_pipeline_hash_still_schema_based(self, tag_schema, data_schema): - """pipeline_hash stays schema-based even when concrete is bound.""" - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - from orcapod.core.sources.dict_source import DictSource - - src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src - ) - assert job_node.pipeline_hash() == node.pipeline_hash() - - def test_as_node_returns_source_node(self, tag_schema, data_schema): - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - - job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - node = job_node.as_node() - assert isinstance(node, SourceNode) - assert node.content_hash() == job_node.content_hash() - - def test_mutable_concrete_updates_in_place(self, tag_schema, data_schema): - """Binding concrete mutates _concrete in-place.""" - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.core.sources.dict_source import DictSource - - job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - assert job_node._concrete is None - src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - job_node._concrete = src - assert job_node._concrete is src -``` - -- [ ] **Step 1.3: Run tests to confirm failure** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py -v 2>&1 | tail -20 -``` - -Expected: ImportError or AttributeError (SourceNode doesn't yet have `name=` constructor arg) - -- [ ] **Step 1.4: Rewrite src/orcapod/core/nodes/source_node.py** - -Replace the entire file content: - -```python -"""Source node hierarchy for Pipeline and PipelineJob. - -SourceNode — schema-only input-slot declaration (replaces SourceSpec). -SourceJobNode — execution variant that wraps a concrete StreamProtocol. -Both share SourceNodeBase which provides hash-stable identity. - -Hash-stability guarantee: - SourceNode(name=n, tag_schema=t, data_schema=d).content_hash() - == SourceSpec(name=n, tag_schema=t, data_schema=d).content_hash() - -This is achieved by using identical identity_structure(): - ("SourceSpec", name, tag_schema, data_schema) -""" -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from collections.abc import Iterator -from typing import TYPE_CHECKING, Any - -from orcapod import contexts -from orcapod.config import DEFAULT_CONFIG -from orcapod.core.base import TraceableBase -from orcapod.errors import SourceSpecMismatchError, UnboundSourceError -from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -from orcapod.types import ColumnConfig, ContentHash, Schema - -if TYPE_CHECKING: - import pyarrow as pa - - from orcapod.protocols.core_protocols import StreamProtocol - -logger = logging.getLogger(__name__) - - -class SourceNodeBase(TraceableBase, ABC): - """Abstract base for SourceNode and SourceJobNode. - - Provides schema-based identity (content_hash, pipeline_hash) and - shared properties. Both sub-types carry identical schemas so their - pipeline_hash() values always match; content_hash() diverges only - when SourceJobNode has a concrete source bound. - - Args: - name: The input-slot name used as the key in - ``PipelineJob.bind(sources={name: source})``. - tag_schema: Mapping of tag column names to Python types. - data_schema: Mapping of data column names to Python types. - data_context: Optional data context override. - """ - - node_type = "source" - - def __init__( - self, - name: str, - tag_schema: Schema, - data_schema: Schema, - data_context: str | contexts.DataContext | None = None, - ) -> None: - super().__init__(data_context=data_context) - self._name = name - self._tag_schema = tag_schema - self._data_schema = data_schema - - # ------------------------------------------------------------------ - # Identity — hash-stable against old SourceSpec - # ------------------------------------------------------------------ - - def identity_structure(self) -> Any: - """Return the content identity: ``("SourceSpec", name, tag_schema, data_schema)``. - - Deliberately matches ``SourceSpec.identity_structure()`` so that a - ``SourceNode`` constructed with the same arguments as a ``SourceSpec`` - produces an identical ``content_hash()``. This preserves all DB paths - computed from pre-refactor pipelines. - """ - return ("SourceSpec", self._name, self._tag_schema, self._data_schema) - - def pipeline_identity_structure(self) -> Any: - """Return the pipeline identity: ``(tag_schema, data_schema)`` (name-independent). - - Matches ``RootSource.pipeline_identity_structure()`` so that sources - with identical schemas share the same DB table paths regardless of name. - """ - return (self._tag_schema, self._data_schema) - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def name(self) -> str: - """The input-slot name used as the key in ``PipelineJob.bind(sources={...})``.""" - return self._name - - def computed_label(self) -> str | None: - """Resolve the node label to the slot name. - - Implements ``LabelableMixin.computed_label()`` so that ``self.label`` - resolves to the slot name without an explicit label assignment. - - Returns: - The slot name. - """ - return self._name - - @property - def tag_schema(self) -> Schema: - """Tag schema for this input slot.""" - return self._tag_schema - - @property - def data_schema(self) -> Schema: - """Data schema for this input slot.""" - return self._data_schema - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return ``(tag_schema, data_schema)``. - - Args: - columns: Ignored. - all_info: Ignored. - - Returns: - Tuple of ``(tag_schema, data_schema)``. - """ - return (self._tag_schema, self._data_schema) - - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Return ``(tag_keys, data_keys)``. - - Args: - columns: Ignored. - all_info: Ignored. - - Returns: - Tuple of ``(tag_column_names, data_column_names)``. - """ - return (tuple(self._tag_schema.keys()), tuple(self._data_schema.keys())) - - # ------------------------------------------------------------------ - # Validation - # ------------------------------------------------------------------ - - def validate(self, source: StreamProtocol) -> None: - """Check that *source* is schema-compatible with this node's declared schema. - - Args: - source: A concrete stream to validate. - - Raises: - SourceSpecMismatchError: If schema columns don't match. - """ - source_tag, source_data = source.output_schema() - - tag_issues: list[str] = [] - data_issues: list[str] = [] - - spec_tag_cols = set(self._tag_schema.keys()) - src_tag_cols = set(source_tag.keys()) - if spec_tag_cols != src_tag_cols: - missing = spec_tag_cols - src_tag_cols - extra = src_tag_cols - spec_tag_cols - if missing: - tag_issues.append(f"missing tag columns: {sorted(missing)}") - if extra: - tag_issues.append(f"unexpected tag columns: {sorted(extra)}") - - spec_data_cols = set(self._data_schema.keys()) - src_data_cols = set(source_data.keys()) - if spec_data_cols != src_data_cols: - missing = spec_data_cols - src_data_cols - extra = src_data_cols - spec_data_cols - if missing: - data_issues.append(f"missing data columns: {sorted(missing)}") - if extra: - data_issues.append(f"unexpected data columns: {sorted(extra)}") - - if tag_issues or data_issues: - raise SourceSpecMismatchError( - f"SourceNode '{self._name}' is not compatible with the provided source. " - + "; ".join(tag_issues + data_issues) - ) - - # ------------------------------------------------------------------ - # Abstract - # ------------------------------------------------------------------ - - @abstractmethod - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Yield ``(tag, data)`` pairs, or raise if data is unavailable.""" - ... - - def __repr__(self) -> str: - return ( - f"{type(self).__name__}(name={self._name!r}, " - f"tag_schema={dict(self._tag_schema)!r}, " - f"data_schema={dict(self._data_schema)!r})" - ) - - -class SourceNode(SourceNodeBase): - """Schema-only input-slot declaration for ``Pipeline`` recording. - - Replaces ``SourceSpec`` as the user-facing way to declare typed pipeline - inputs. Pass a ``SourceNode`` inside a ``with pipeline:`` block as the - upstream for any pod invocation. - - Example:: - - slot = SourceNode(name="data", tag_schema={"id": int}, data_schema={"v": float}) - with pipeline: - result = my_pod(slot) - - job = PipelineJob.from_pipeline(pipeline, store=db, sources={"data": my_source}) - job.run() - - Hash-stability note: - ``identity_structure()`` returns ``("SourceSpec", name, tag_schema, data_schema)`` - — identical to the old ``SourceSpec`` — so existing DB paths remain valid. - """ - - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Raise ``UnboundSourceError`` — ``SourceNode`` carries no data. - - Raises: - UnboundSourceError: Always. - """ - raise UnboundSourceError( - f"SourceNode '{self._name}' is not bound to a concrete source. " - "Use PipelineJob.from_pipeline(..., sources={'': source}) " - "or job.bind(sources={'': source}) to attach data." - ) - - -class SourceJobNode(SourceNodeBase): - """Execution-ready source node wrapping an optional concrete stream. - - Used inside ``PipelineJob._persistent_node_map``. The ``_concrete`` - field is **mutable** — ``PipelineJob.bind(sources={...})`` updates it - in-place so that downstream ``FunctionJobNode`` objects (which hold a - reference to this same object) automatically see the new concrete source - without cascading reference updates. - - Hash behaviour: - - * ``content_hash()`` — delegates to ``_concrete.content_hash()`` when - bound; falls back to schema-based ``SourceNodeBase.content_hash()`` (== - ``SourceNode.content_hash()``) when unbound. - * ``pipeline_hash()`` — always schema-based (inherited); never - data-inclusive. This invariant keeps DB paths stable across different - data sources bound to the same slot. - - Args: - name: Slot name. - tag_schema: Tag schema. - data_schema: Data schema. - concrete: Optional concrete stream. Can be set or replaced later via - ``job_node._concrete = source``. - data_context: Optional data context override. - """ - - def __init__( - self, - name: str, - tag_schema: Schema, - data_schema: Schema, - concrete: StreamProtocol | None = None, - data_context: str | contexts.DataContext | None = None, - ) -> None: - super().__init__( - name=name, - tag_schema=tag_schema, - data_schema=data_schema, - data_context=data_context, - ) - self._concrete: StreamProtocol | None = concrete - - def content_hash(self, hasher=None) -> ContentHash: - """Return data-inclusive hash when bound; schema-based hash when unbound. - - Args: - hasher: Optional semantic hasher. - - Returns: - ``_concrete.content_hash(hasher)`` when bound, otherwise - ``SourceNodeBase.content_hash(hasher)``. - """ - if self._concrete is not None: - if hasher is None: - hasher = self.data_context.semantic_hasher - return self._concrete.content_hash(hasher) - return super().content_hash(hasher) - - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Delegate to concrete source, or raise if unbound. - - Raises: - UnboundSourceError: When no concrete source is attached. - """ - if self._concrete is None: - raise UnboundSourceError( - f"SourceJobNode '{self._name}' has no concrete source bound. " - "Call job.bind(sources={'': source}) before running." - ) - return self._concrete.iter_data() - - def as_node(self) -> SourceNode: - """Return the lightweight ``SourceNode`` equivalent of this job node. - - Returns: - A new ``SourceNode`` with the same name and schemas. - """ - return SourceNode( - name=self._name, - tag_schema=self._tag_schema, - data_schema=self._data_schema, - ) -``` - -- [ ] **Step 1.5: Run new source_node tests** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py -v -``` - -Expected: All 14 tests pass. - -- [ ] **Step 1.6: Update nodes/__init__.py to export new types** - -Replace `src/orcapod/core/nodes/__init__.py` content: - -```python -from typing import TypeAlias - -from .function_node import FunctionNode -from .operator_node import OperatorNode -from .source_node import SourceJobNode, SourceNode, SourceNodeBase - -GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode - -__all__ = [ - "FunctionNode", - "GraphNode", - "OperatorNode", - "SourceJobNode", - "SourceNode", - "SourceNodeBase", -] -``` - -- [ ] **Step 1.7: Update Pipeline.compile() to accept SourceNode directly** - -In `src/orcapod/pipeline/graph.py`, find the `compile()` method leaf-handling block (around line 218) and replace: - -```python -# OLD — lines ~218-229 -from orcapod.core.sources.source_spec import SourceSpec -stream = self._upstreams[node_hash] -if not isinstance(stream, SourceSpec): - raise ValueError( - f"Pipeline: all leaf inputs must be SourceSpec instances, " - f"but found {type(stream).__name__!r}. " - "Use 'with PipelineJob:' to record a pipeline with concrete sources, " - "or replace concrete sources with SourceSpec declarations." - ) -node = SourceNode(stream=stream) -``` - -With: - -```python -# NEW -from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass -stream = self._upstreams[node_hash] -if not isinstance(stream, SourceNodeClass): - raise ValueError( - f"Pipeline: all leaf inputs must be SourceNode instances, " - f"but found {type(stream).__name__!r}. " - "Use 'with PipelineJob:' to record a pipeline with concrete sources, " - "or replace concrete sources with SourceNode declarations." - ) -node = stream # SourceNode IS the leaf — no wrapping needed -``` - -Also update the `isinstance` checks later in `compile()` that reference `SourceNode` for node_type annotation — they now just check `isinstance(node, SourceNodeClass)`. - -- [ ] **Step 1.8: Update PipelineJob._ensure_spec → _ensure_source_node** - -In `src/orcapod/pipeline/job.py`: - -1. Rename `_ensure_spec` to `_ensure_source_node` and update it: - -```python -def _ensure_source_node(self, source: cp.StreamProtocol) -> "SourceNode": - """Promote *source* to a SourceNode, storing the concrete binding. - - If the slot already exists (same label/hash key), returns the cached node. - - Args: - source: A concrete ``RootSource`` to promote. - - Returns: - The ``SourceNode`` slot declaration for this source. - """ - from orcapod.core.nodes.source_node import SourceNode - - has_label = source.has_assigned_label - if has_label: - name = source.label # type: ignore[attr-defined] - else: - name = source.content_hash().to_string() - - if name not in self._spec_by_name: - tag_schema, data_schema = source.output_schema() - node = SourceNode(name=name, tag_schema=tag_schema, data_schema=data_schema) - self._spec_by_name[name] = node # type: ignore[assignment] - self._sources[name] = source - return self._spec_by_name[name] # type: ignore[return-value] -``` - -2. Update `_is_concrete_source` to not exclude `SourceNode`: - -```python -@staticmethod -def _is_concrete_source(stream: cp.StreamProtocol) -> bool: - """True if *stream* is a concrete RootSource (not a SourceNode).""" - from orcapod.core.sources.base import RootSource - from orcapod.core.nodes.source_node import SourceNode - - return isinstance(stream, RootSource) and not isinstance(stream, SourceNode) -``` - -3. Rename `_to_spec_stream` → `_to_node_stream` and update: - -```python -def _to_node_stream(self, stream: cp.StreamProtocol) -> cp.StreamProtocol: - """Convert *stream* to a node-based equivalent for consistent hash recording. - - Concrete ``RootSource`` instances are promoted to ``SourceNode`` via - ``_ensure_source_node``. ``DynamicPodStream`` instances have their - upstreams recursively converted. - - Args: - stream: The upstream stream to convert. - - Returns: - A node-based stream with a stable hash for recording. - """ - from orcapod.core.operators.static_output_pod import DynamicPodStream - - if self._is_concrete_source(stream): - return self._ensure_source_node(stream) - if isinstance(stream, DynamicPodStream): - node_upstreams = tuple(self._to_node_stream(s) for s in stream.upstreams) - return DynamicPodStream( - pod=stream._pod, - upstreams=node_upstreams, - label=stream._label, - ) - return stream -``` - -4. Update `record_function_pod_invocation` to call `_to_node_stream`: - -```python -def record_function_pod_invocation(self, pod, input_stream, label=None): - from orcapod.core.nodes import FunctionNode - input_stream = self._to_node_stream(input_stream) - input_hash = input_stream.content_hash().to_string() - function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) - fn_hash = function_node.content_hash().to_string() - self._rec_node_lut[fn_hash] = function_node - self._rec_upstreams[input_hash] = input_stream - self._rec_graph_edges.append((input_hash, fn_hash)) -``` - -5. Update `record_operator_pod_invocation` to call `_to_node_stream`: - -```python -def record_operator_pod_invocation(self, pod, upstreams=(), label=None): - from orcapod.core.nodes import OperatorNode - processed = tuple(self._to_node_stream(s) for s in upstreams) - operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) - op_hash = operator_node.content_hash().to_string() - self._rec_node_lut[op_hash] = operator_node - for upstream in processed: - up_hash = upstream.content_hash().to_string() - self._rec_upstreams[up_hash] = upstream - self._rec_graph_edges.append((up_hash, op_hash)) -``` - -6. Update `unbound_specs` → `unbound_source_nodes`: - -```python -def unbound_source_nodes(self) -> list["SourceNode"]: - """Return all SourceNode slots not yet bound in this job. - - Returns: - List of unbound ``SourceNode`` instances, in graph order. - """ - from orcapod.core.nodes.source_node import SourceNode - - if self._compiled_pipeline is None: - return [] - - unbound: list[SourceNode] = [] - seen: set[str] = set() - for node in self._compiled_pipeline._persistent_node_map.values(): - if ( - isinstance(node, SourceNode) - and node.name not in self._sources - and node.name not in seen - ): - unbound.append(node) - seen.add(node.name) - return unbound -``` - -Keep `unbound_specs` as a deprecated alias that calls `unbound_source_nodes` (will be removed in Task 8): - -```python -def unbound_specs(self): - """Deprecated — use unbound_source_nodes() instead.""" - return self.unbound_source_nodes() -``` - -7. Update `is_complete` to use `unbound_source_nodes`: - -```python -def is_complete(self) -> bool: - return self._store is not None and len(self.unbound_source_nodes()) == 0 -``` - -8. Update `is_runnable` to use `SourceNode` instead of `SourceSpec`: - -```python -def is_runnable(self, node_label: str) -> bool: - from orcapod.core.nodes.source_node import SourceNode - - pipeline = self._compiled_pipeline - if pipeline is None: - return False - target = pipeline._nodes.get(node_label) - if target is None: - return False - if pipeline._node_graph is None: - return False - - import networkx as nx - - for node in nx.ancestors(pipeline._node_graph, target) | {target}: - if isinstance(node, SourceNode) and node.name not in self._sources: - return False - return True -``` - -9. Update `bind()` validation: - -```python -def bind(self, sources=None, store=None, execution_context=None) -> "PipelineJob": - from orcapod.core.nodes.source_node import SourceNode - - merged_sources = dict(self._sources) - if sources is not None: - pipeline = self._compiled_pipeline - if pipeline is not None: - spec_names = { - node.name - for node in pipeline._persistent_node_map.values() - if isinstance(node, SourceNode) - } - for name, source in sources.items(): - for node in pipeline._persistent_node_map.values(): - if isinstance(node, SourceNode) and node.name == name: - node.validate(source) - break - unknown = set(sources.keys()) - spec_names - if unknown: - raise ValueError( - f"bind() received source keys with no matching SourceNode in the pipeline: " - f"{sorted(unknown)}. Known names: {sorted(spec_names)}" - ) - merged_sources.update(sources) - - return PipelineJob( - name=self._pipeline_name, - store=store if store is not None else self._store, - execution_context=( - execution_context if execution_context is not None else self._execution_context - ), - _pipeline=self._compiled_pipeline, - sources=merged_sources, - ) -``` - -- [ ] **Step 1.9: Run pipeline test suite** - -```bash -uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: All existing pipeline tests pass. Failures here indicate missed `SourceSpec` references — fix each one. - -- [ ] **Step 1.10: Commit** - -```bash -git add src/orcapod/errors.py \ - src/orcapod/core/nodes/source_node.py \ - src/orcapod/core/nodes/__init__.py \ - src/orcapod/pipeline/graph.py \ - src/orcapod/pipeline/job.py \ - tests/test_core/nodes/__init__.py \ - tests/test_core/nodes/test_source_node.py -git commit -m "refactor(nodes): replace SourceSpec with schema-only SourceNode + add SourceJobNode" -``` - ---- - -## Task 2: Split function_node.py into FunctionNodeBase + FunctionNode + FunctionJobNode - -**Files:** -- Rewrite: `src/orcapod/core/nodes/function_node.py` -- Modify: `src/orcapod/core/nodes/__init__.py` -- Create: `tests/test_core/nodes/test_function_node_split.py` - -**Class hierarchy (both siblings inherit from the same base — neither inherits from the other):** - -``` -FunctionNodeBase(StreamBase) -│ __init__(function_pod, input_stream, label, tracker_manager, config, table_scope) -│ Shared: load_status, producer, data_context, data_context_key, executor, -│ upstreams property/setter, content_hash() override (read-only mode), -│ pipeline_hash() override (read-only mode), output_schema(), keys(), -│ node_identity_path, node_uri, clear_cache() (identity cache only) -│ -├── FunctionNode(FunctionNodeBase) -│ __init__: same params as base, no DB params -│ iter_data(): raises PipelineJobRequiredError -│ as_node(): returns self -│ -└── FunctionJobNode(FunctionNodeBase) - __init__: adds pipeline_database, result_database params - Additional state: _pipeline_database, _cached_function_pod, _output_schema_hash, - _cached_output_datas, _cached_output_table, _cached_content_hash_column - attach_databases(), _require_pipeline_database(), _filter_by_content_hash() - from_descriptor() classmethod - clear_cache() override: clears DB cache state too - execute(), execute_data(), iter_data() (two-phase), async methods, - get_cached_results(), as_source() - as_node(): returns FunctionNode(function_pod, input_stream, label, table_scope) -``` - -**Split strategy for the 1466-line current `FunctionNode`:** -- `FunctionNodeBase` gets everything from the current `__init__` that is *not* DB state, plus all shared properties (lines ~76–534 of current file minus DB fields). -- `FunctionNode` is a new thin class with no extra state. -- `FunctionJobNode` gets the remaining DB state + all execution methods (lines ~530–1466 plus DB fields from `__init__`). - -- [ ] **Step 2.1: Write failing tests** - -Create `tests/test_core/nodes/test_function_node_split.py`: - -```python -"""Tests for the FunctionNode / FunctionJobNode split.""" -from __future__ import annotations - -import pytest - -from orcapod.errors import PipelineJobRequiredError -from orcapod.types import Schema - - -@pytest.fixture -def simple_pipeline(tmp_path): - """A minimal in-memory store + function pod + source node fixture.""" - from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.dict_source import DictSource - from orcapod.databases.in_memory_database import InMemoryDatabase - - tag_schema = Schema({"id": int}) - data_schema = Schema({"value": float}) - - source_node = SourceNode(name="src", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - db = InMemoryDatabase() - - @FunctionPod.from_function - def double(value: float) -> dict: - return {"result": value * 2} - - return { - "source_node": source_node, - "source": src, - "db": db, - "pod": double, - "tag_schema": tag_schema, - "data_schema": data_schema, - } - - -class TestThinFunctionNode: - def test_iter_data_raises_pipeline_job_required(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionNode - - fn = FunctionNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - with pytest.raises(PipelineJobRequiredError): - list(fn.iter_data()) - - def test_content_hash_is_stable(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionNode - - fn = FunctionNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - h1 = fn.content_hash() - h2 = fn.content_hash() - assert h1 == h2 - - def test_node_type(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionNode - - fn = FunctionNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - assert fn.node_type == "function" - - def test_output_schema(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionNode - - fn = FunctionNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - tag_s, data_s = fn.output_schema() - assert "result" in data_s - - -class TestFunctionJobNodeHashParity: - """FunctionJobNode must have identical content_hash / pipeline_hash to FunctionNode.""" - - def test_content_hash_matches_function_node(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode - - fn = FunctionNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - fjn = FunctionJobNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - assert fn.content_hash() == fjn.content_hash() - - def test_pipeline_hash_matches_function_node(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode - - fn = FunctionNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - fjn = FunctionJobNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - assert fn.pipeline_hash() == fjn.pipeline_hash() - - def test_as_node_returns_function_node(self, simple_pipeline): - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode - - fjn = FunctionJobNode( - function_pod=simple_pipeline["pod"], - input_stream=simple_pipeline["source_node"], - ) - fn = fjn.as_node() - assert isinstance(fn, FunctionNode) - assert fn.content_hash() == fjn.content_hash() -``` - -- [ ] **Step 2.2: Run tests to confirm failure** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_split.py -v 2>&1 | tail -20 -``` - -Expected: `ImportError: cannot import name 'FunctionJobNode'` - -- [ ] **Step 2.3: Rewrite function_node.py with proper base class** - -Restructure `src/orcapod/core/nodes/function_node.py` into three classes. Keep all existing -logic intact — just redistribute it. The file will still be long; the important thing is -clean ownership of state and methods. - -**a) `FunctionNodeBase(StreamBase)` — new class, replaces the top of the current `FunctionNode`** - -Carries everything that does not touch a database. Its `__init__` is the current -`FunctionNode.__init__` with the DB parameters and DB-state initialisation removed: - -```python -class FunctionNodeBase(StreamBase): - """Shared identity, properties, and schema logic for FunctionNode and FunctionJobNode. - - Neither subtype is a subtype of the other — both inherit directly from this base. - This class carries no database references. - """ - - node_type = "function" - - def __init__( - self, - function_pod: FunctionPodProtocol, - input_stream: StreamProtocol, - tracker_manager: TrackerManagerProtocol | None = None, - label: str | None = None, - config: Config | None = None, - table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", - ) -> None: - if tracker_manager is None: - tracker_manager = DEFAULT_TRACKER_MANAGER - self.tracker_manager = tracker_manager - self._data_function = function_pod.data_function - self._function_pod = function_pod - super().__init__(label=label, config=config) - - # Schema validation (skip for UNAVAILABLE streams) - from orcapod.pipeline.serialization import LoadStatus - _stream_unavailable = ( - hasattr(input_stream, "load_status") - and input_stream.load_status == LoadStatus.UNAVAILABLE - ) - if not _stream_unavailable: - _, incoming_data_types = input_stream.output_schema() - expected_data_schema = self._data_function.input_data_schema - if not schema_utils.check_schema_compatibility( - incoming_data_types, expected_data_schema - ): - raise ValueError( - f"Incoming data type {incoming_data_types} from {input_stream} " - f"is not compatible with expected input schema {expected_data_schema}" - ) - - self._input_stream = input_stream - - if table_scope not in ("pipeline_hash", "content_hash"): - raise ValueError( - f"Unknown table_scope {table_scope!r}. " - "Expected one of: 'pipeline_hash', 'content_hash'." - ) - self._table_scope = table_scope - self._node_identity_path_cache: tuple[str, ...] | None = None - - # Descriptor fields for read-only / UNAVAILABLE deserialized nodes - from orcapod.pipeline.serialization import LoadStatus - self._load_status: LoadStatus = LoadStatus.FULL - self._stored_content_hash: str | None = None - self._stored_pipeline_hash: str | None = None - self._stored_schema: dict = {} - self._stored_node_uri: tuple[str, ...] = () - self._stored_pipeline_path: tuple[str, ...] = () - self._stored_result_record_path: tuple[str, ...] = () - self._descriptor: dict = {} - - # Copy these shared properties verbatim from the current FunctionNode: - # load_status, producer, data_context, data_context_key, executor (+ setter), - # upstreams (+ setter), content_hash() override, pipeline_hash() override, - # output_schema(), keys(), node_identity_path, node_uri - # - # clear_cache() in base clears only the identity path cache: - - def clear_cache(self) -> None: - self._node_identity_path_cache = None - self._update_modified_time() -``` - -**b) `FunctionNode(FunctionNodeBase)` — thin blueprint node** - -```python -class FunctionNode(FunctionNodeBase): - """Lightweight blueprint node for ``Pipeline`` recording. - - Carries no database references. ``iter_data()`` raises - ``PipelineJobRequiredError``. Use ``PipelineJob.from_pipeline()`` to - obtain an executable ``FunctionJobNode``. - - Args: - function_pod: The wrapped function pod. - input_stream: Upstream stream (``SourceNode`` or another blueprint node). - tracker_manager: Optional tracker manager override. - label: Optional display label. - config: Optional node config. - table_scope: DB table scoping strategy (preserved for hash stability when - later converted to a ``FunctionJobNode``). - """ - - def iter_data(self): - """Raise PipelineJobRequiredError — blueprint nodes cannot produce data. - - Raises: - PipelineJobRequiredError: Always. - """ - from orcapod.errors import PipelineJobRequiredError - - raise PipelineJobRequiredError( - f"FunctionNode '{self.label}' is a blueprint node and cannot produce data. " - "Wrap the containing Pipeline in a PipelineJob to execute:\n" - " job = PipelineJob.from_pipeline(pipeline, store=db, sources={...})\n" - " job.run()" - ) - - def as_node(self) -> "FunctionNode": - """Return self — already a lightweight blueprint node.""" - return self -``` - -**c) `FunctionJobNode(FunctionNodeBase)` — DB-backed execution node** - -```python -class FunctionJobNode(FunctionNodeBase): - """DB-backed execution node for ``PipelineJob`` graphs. - - Adds database references and all execution logic on top of - ``FunctionNodeBase``. ``FunctionNode`` and ``FunctionJobNode`` - are siblings — neither inherits from the other. - - Args: - function_pod: The wrapped function pod. - input_stream: Upstream stream (``SourceJobNode`` or another job node). - tracker_manager: Optional tracker manager override. - label: Optional display label. - config: Optional node config. - table_scope: DB table scoping strategy. - pipeline_database: Optional database for pipeline records. - result_database: Optional database for cached results. - """ - - def __init__( - self, - function_pod: FunctionPodProtocol, - input_stream: StreamProtocol, - tracker_manager: TrackerManagerProtocol | None = None, - label: str | None = None, - config: Config | None = None, - table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", - pipeline_database: ArrowDatabaseProtocol | None = None, - result_database: ArrowDatabaseProtocol | None = None, - ) -> None: - super().__init__( - function_pod=function_pod, - input_stream=input_stream, - tracker_manager=tracker_manager, - label=label, - config=config, - table_scope=table_scope, - ) - # DB-specific state - self._cached_output_datas: dict[str, tuple[TagProtocol, DataProtocol | None]] = {} - self._cached_output_table: pa.Table | None = None - self._cached_content_hash_column: pa.Array | None = None - self._pipeline_database: ArrowDatabaseProtocol | None = None - self._cached_function_pod: CachedFunctionPod | None = None - self._output_schema_hash: str | None = None - - if pipeline_database is not None: - self.attach_databases( - pipeline_database=pipeline_database, - result_database=result_database, - ) - - def clear_cache(self) -> None: - """Clear all caches including DB-backed output cache.""" - self._cached_output_datas.clear() - self._cached_output_table = None - self._cached_content_hash_column = None - self._node_identity_path_cache = None - self._update_modified_time() - - def as_node(self) -> FunctionNode: - """Return a lightweight ``FunctionNode`` with the same identity. - - Returns: - A new ``FunctionNode`` carrying the same function_pod, input_stream, - label, and table_scope — and therefore identical content_hash() and - pipeline_hash() values. - """ - return FunctionNode( - function_pod=self._function_pod, - input_stream=self._input_stream, - label=self._label, - table_scope=self._table_scope, - ) - - # Move these verbatim from the current FunctionNode (they already exist there): - # attach_databases(), _require_pipeline_database(), _filter_by_content_hash(), - # from_descriptor() classmethod, execute_data(), execute(), iter_data() (two-phase), - # _process_data_internal(), get_cached_results(), as_source(), async methods. -``` - -> **Implementation note:** The bulk of `FunctionJobNode` is a verbatim move of the current -> `FunctionNode`'s DB methods. No logic changes are needed — only re-homing the code into -> the new class and adjusting the `__init__` to call `super().__init__()` without DB params. -> Any place that currently says `isinstance(node, FunctionNode)` in execution-path code -> should be updated to `isinstance(node, FunctionJobNode)` where the intent is "DB-capable -> node"; use `isinstance(node, FunctionNodeBase)` where the intent is "any function node". - -- [ ] **Step 2.4: Run new tests** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_split.py -v -``` - -Expected: All 7 tests pass. - -- [ ] **Step 2.5: Verify isinstance hierarchy is correct** - -```bash -uv run python -c " -from orcapod.core.nodes.function_node import FunctionNode, FunctionJobNode, FunctionNodeBase -from orcapod.core.streams.base import StreamBase - -# Neither should be a subtype of the other -assert not issubclass(FunctionNode, FunctionJobNode), 'FunctionNode must NOT inherit from FunctionJobNode' -assert not issubclass(FunctionJobNode, FunctionNode), 'FunctionJobNode must NOT inherit from FunctionNode' - -# Both should share the base -assert issubclass(FunctionNode, FunctionNodeBase) -assert issubclass(FunctionJobNode, FunctionNodeBase) -assert issubclass(FunctionNodeBase, StreamBase) -print('isinstance hierarchy: CORRECT') -" -``` - -Expected: `isinstance hierarchy: CORRECT` - -- [ ] **Step 2.6: Update nodes/__init__.py** - -```python -from typing import TypeAlias - -from .function_node import FunctionJobNode, FunctionNode, FunctionNodeBase -from .operator_node import OperatorNode -from .source_node import SourceJobNode, SourceNode, SourceNodeBase - -GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode -JobNode: TypeAlias = SourceJobNode | FunctionJobNode - -__all__ = [ - "FunctionJobNode", - "FunctionNode", - "FunctionNodeBase", - "GraphNode", - "JobNode", - "OperatorNode", - "SourceJobNode", - "SourceNode", - "SourceNodeBase", -] -``` - -- [ ] **Step 2.7: Run full pipeline test suite** - -```bash -uv run pytest tests/test_pipeline/ tests/test_core/ -v --tb=short 2>&1 | tail -40 -``` - -Any test that instantiated `FunctionNode(..., pipeline_database=...)` directly will now -fail because `FunctionNode` no longer accepts DB params. Update those call sites to use -`FunctionJobNode` instead. - -- [ ] **Step 2.8: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py \ - src/orcapod/core/nodes/__init__.py \ - tests/test_core/nodes/test_function_node_split.py -git commit -m "refactor(nodes): split FunctionNode → FunctionNodeBase + FunctionNode + FunctionJobNode" -``` - ---- - -## Task 3: Split operator_node.py into OperatorNodeBase + OperatorNode + OperatorJobNode - -**Files:** -- Rewrite: `src/orcapod/core/nodes/operator_node.py` -- Modify: `src/orcapod/core/nodes/__init__.py` -- Create: `tests/test_core/nodes/test_operator_node_split.py` - -**Class hierarchy (same sibling pattern as Task 2 — neither inherits from the other):** - -``` -OperatorNodeBase(StreamBase) -│ __init__(operator, input_streams, label, tracker_manager, config, table_scope) -│ Shared: load_status, identity_structure(), pipeline_identity_structure(), -│ content_hash() override, pipeline_hash() override, producer, -│ data_context, data_context_key, upstreams property/setter, -│ keys(), output_schema(), node_identity_path, node_uri -│ -├── OperatorNode(OperatorNodeBase) -│ iter_data(): raises PipelineJobRequiredError -│ as_node(): returns self -│ -└── OperatorJobNode(OperatorNodeBase) - __init__: adds pipeline_database, cache_mode params - Additional state: _pipeline_database, _cache_mode, - _cached_output_stream, _cached_output_table - attach_databases(), from_descriptor() classmethod, - execute(), run(), iter_data() (cache-mode aware), - as_table(), get_all_records(), as_source(), async methods - as_node(): returns OperatorNode(operator, input_streams, label, table_scope) -``` - -**Split strategy for the current 903-line `OperatorNode`:** -- `OperatorNodeBase` gets everything from `__init__` that is not DB state, plus all shared properties/identity methods. -- `OperatorNode` is a thin class with no extra state. -- `OperatorJobNode` gets the DB state + all execution methods. - -- [ ] **Step 3.1: Write failing tests** - -Create `tests/test_core/nodes/test_operator_node_split.py`: - -```python -"""Tests for the OperatorNode / OperatorJobNode split.""" -from __future__ import annotations - -import pytest - -from orcapod.errors import PipelineJobRequiredError -from orcapod.types import Schema - - -@pytest.fixture -def source_pair(): - from orcapod.core.nodes.source_node import SourceNode - - tag_schema = Schema({"id": int}) - data_schema_a = Schema({"a": float}) - data_schema_b = Schema({"b": float}) - node_a = SourceNode(name="src_a", tag_schema=tag_schema, data_schema=data_schema_a) - node_b = SourceNode(name="src_b", tag_schema=tag_schema, data_schema=data_schema_b) - return node_a, node_b - - -class TestThinOperatorNode: - def test_iter_data_raises_pipeline_job_required(self, source_pair): - from orcapod.core.nodes.operator_node import OperatorNode - from orcapod.core.operators.join import Join - - op = Join() - node_a, node_b = source_pair - op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) - with pytest.raises(PipelineJobRequiredError): - list(op_node.iter_data()) - - def test_node_type(self, source_pair): - from orcapod.core.nodes.operator_node import OperatorNode - from orcapod.core.operators.join import Join - - op = Join() - node_a, node_b = source_pair - op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) - assert op_node.node_type == "operator" - - -class TestOperatorJobNodeHashParity: - def test_content_hash_matches_operator_node(self, source_pair): - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode - from orcapod.core.operators.join import Join - - op = Join() - node_a, node_b = source_pair - thin = OperatorNode(operator=op, input_streams=(node_a, node_b)) - job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) - assert thin.content_hash() == job.content_hash() - - def test_pipeline_hash_matches_operator_node(self, source_pair): - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode - from orcapod.core.operators.join import Join - - op = Join() - node_a, node_b = source_pair - thin = OperatorNode(operator=op, input_streams=(node_a, node_b)) - job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) - assert thin.pipeline_hash() == job.pipeline_hash() - - def test_as_node_returns_operator_node(self, source_pair): - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode - from orcapod.core.operators.join import Join - - op = Join() - node_a, node_b = source_pair - job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) - thin = job.as_node() - assert isinstance(thin, OperatorNode) - assert thin.content_hash() == job.content_hash() -``` - -- [ ] **Step 3.2: Run tests to confirm failure** - -```bash -uv run pytest tests/test_core/nodes/test_operator_node_split.py -v 2>&1 | tail -10 -``` - -Expected: `ImportError: cannot import name 'OperatorJobNode'` - -- [ ] **Step 3.3: Rewrite operator_node.py with proper base class** - -Restructure `src/orcapod/core/nodes/operator_node.py` into three classes following the -same pattern as Task 2. - -**a) `OperatorNodeBase(StreamBase)` — new class** - -```python -class OperatorNodeBase(StreamBase): - """Shared identity, properties, and schema logic for OperatorNode and OperatorJobNode. - - Neither subtype is a subtype of the other — both inherit directly from this base. - This class carries no database references. - """ - - node_type = "operator" - HASH_COLUMN_NAME = "_record_hash" - - def __init__( - self, - operator: OperatorPodProtocol, - input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], - tracker_manager: TrackerManagerProtocol | None = None, - label: str | None = None, - config: Config | None = None, - table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", - ) -> None: - if tracker_manager is None: - tracker_manager = DEFAULT_TRACKER_MANAGER - self.tracker_manager = tracker_manager - self._operator = operator - self._input_streams = tuple(input_streams) - super().__init__(label=label, config=config) - - # Eager input validation - self._operator.validate_inputs(*self._input_streams) - - if table_scope not in ("pipeline_hash", "content_hash"): - raise ValueError( - f"Unknown table_scope {table_scope!r}. " - "Expected one of: 'pipeline_hash', 'content_hash'." - ) - self._table_scope = table_scope - self._node_identity_path_cache: tuple[str, ...] | None = None - self._set_modified_time(None) - - # Descriptor fields for read-only / UNAVAILABLE deserialized nodes - from orcapod.pipeline.serialization import LoadStatus - self._load_status: LoadStatus = LoadStatus.FULL - self._stored_content_hash: str | None = None - self._stored_pipeline_hash: str | None = None - self._stored_schema: dict = {} - self._stored_node_uri: tuple[str, ...] = () - self._stored_pipeline_path: tuple[str, ...] = () - self._descriptor: dict = {} - - # Copy these verbatim from the current OperatorNode (they already exist there): - # load_status property, identity_structure(), pipeline_identity_structure(), - # content_hash() override, pipeline_hash() override, - # producer, data_context, data_context_key, upstreams property/setter, - # keys(), output_schema(), node_identity_path, node_uri -``` - -**b) `OperatorNode(OperatorNodeBase)` — thin blueprint node** - -```python -class OperatorNode(OperatorNodeBase): - """Lightweight blueprint node for ``Pipeline`` recording. - - Carries no database references. ``iter_data()`` raises - ``PipelineJobRequiredError``. Use ``PipelineJob.from_pipeline()`` to - obtain an executable ``OperatorJobNode``. - """ - - def iter_data(self): - """Raise PipelineJobRequiredError — blueprint nodes cannot produce data. - - Raises: - PipelineJobRequiredError: Always. - """ - from orcapod.errors import PipelineJobRequiredError - - raise PipelineJobRequiredError( - f"OperatorNode '{self.label}' is a blueprint node and cannot produce data. " - "Wrap the containing Pipeline in a PipelineJob to execute:\n" - " job = PipelineJob.from_pipeline(pipeline, store=db, sources={...})\n" - " job.run()" - ) - - def as_node(self) -> "OperatorNode": - """Return self — already a lightweight blueprint node.""" - return self -``` - -**c) `OperatorJobNode(OperatorNodeBase)` — DB-backed execution node** - -```python -class OperatorJobNode(OperatorNodeBase): - """DB-backed execution node for ``PipelineJob`` graphs. - - ``OperatorNode`` and ``OperatorJobNode`` are siblings — neither inherits - from the other. - - Args: - operator: The wrapped operator pod. - input_streams: Upstream job nodes. - tracker_manager: Optional tracker manager override. - label: Optional display label. - config: Optional node config. - table_scope: DB table scoping strategy. - pipeline_database: Optional database for pipeline records. - cache_mode: Caching behaviour (OFF / LOG / REPLAY). - """ - - def __init__( - self, - operator: OperatorPodProtocol, - input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], - tracker_manager: TrackerManagerProtocol | None = None, - label: str | None = None, - config: Config | None = None, - table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", - pipeline_database: ArrowDatabaseProtocol | None = None, - cache_mode: CacheMode = CacheMode.OFF, - ) -> None: - super().__init__( - operator=operator, - input_streams=input_streams, - tracker_manager=tracker_manager, - label=label, - config=config, - table_scope=table_scope, - ) - # DB-specific state - self._cached_output_stream: StreamProtocol | None = None - self._cached_output_table: pa.Table | None = None - self._pipeline_database: ArrowDatabaseProtocol | None = None - self._cache_mode = CacheMode.OFF - - if pipeline_database is not None: - self.attach_databases( - pipeline_database=pipeline_database, - cache_mode=cache_mode, - ) - - def as_node(self) -> OperatorNode: - """Return a lightweight ``OperatorNode`` with the same identity. - - Returns: - A new ``OperatorNode`` carrying the same operator, input_streams, - label, and table_scope — with identical content_hash() and - pipeline_hash() values. - """ - return OperatorNode( - operator=self._operator, - input_streams=self._input_streams, - label=self._label, - table_scope=self._table_scope, - ) - - # Move these verbatim from the current OperatorNode (they already exist there): - # attach_databases(), from_descriptor() classmethod, - # execute(), run(), iter_data() (cache-mode aware), - # as_table(), get_all_records(), as_source(), async methods. -``` - -> **Implementation note:** Same approach as Task 2. The bulk of `OperatorJobNode` is a -> verbatim move of the current `OperatorNode`'s DB methods. Update any `isinstance(node, -> OperatorNode)` in execution-path code to `isinstance(node, OperatorJobNode)` where the -> intent is "DB-capable node"; use `isinstance(node, OperatorNodeBase)` where the intent -> is "any operator node". - -- [ ] **Step 3.4: Verify isinstance hierarchy is correct** - -```bash -uv run python -c " -from orcapod.core.nodes.operator_node import OperatorNode, OperatorJobNode, OperatorNodeBase -from orcapod.core.streams.base import StreamBase - -assert not issubclass(OperatorNode, OperatorJobNode), 'OperatorNode must NOT inherit from OperatorJobNode' -assert not issubclass(OperatorJobNode, OperatorNode), 'OperatorJobNode must NOT inherit from OperatorNode' -assert issubclass(OperatorNode, OperatorNodeBase) -assert issubclass(OperatorJobNode, OperatorNodeBase) -assert issubclass(OperatorNodeBase, StreamBase) -print('isinstance hierarchy: CORRECT') -" -``` - -Expected: `isinstance hierarchy: CORRECT` - -- [ ] **Step 3.5: Update nodes/__init__.py to export all new types** - -```python -from typing import TypeAlias - -from .function_node import FunctionJobNode, FunctionNode, FunctionNodeBase -from .operator_node import OperatorJobNode, OperatorNode, OperatorNodeBase -from .source_node import SourceJobNode, SourceNode, SourceNodeBase - -GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode -JobNode: TypeAlias = SourceJobNode | FunctionJobNode | OperatorJobNode - -__all__ = [ - "FunctionJobNode", - "FunctionNode", - "FunctionNodeBase", - "GraphNode", - "JobNode", - "OperatorJobNode", - "OperatorNode", - "OperatorNodeBase", - "SourceJobNode", - "SourceNode", - "SourceNodeBase", -] -``` - -- [ ] **Step 3.6: Run new and existing tests** - -```bash -uv run pytest tests/test_core/nodes/test_operator_node_split.py tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 -``` - -Any test that instantiated `OperatorNode(..., pipeline_database=...)` directly will now -fail — update those to `OperatorJobNode` instead. - -- [ ] **Step 3.7: Commit** - -```bash -git add src/orcapod/core/nodes/operator_node.py \ - src/orcapod/core/nodes/__init__.py \ - tests/test_core/nodes/test_operator_node_split.py -git commit -m "refactor(nodes): split OperatorNode → OperatorNodeBase + OperatorNode + OperatorJobNode" -``` - ---- - -## Task 4: Remove Pipeline.bind() and add PipelineJob.from_pipeline() + as_pipeline() - -**Files:** -- Modify: `src/orcapod/pipeline/graph.py` — remove `bind()` -- Modify: `src/orcapod/pipeline/job.py` — add `from_pipeline()`, `as_pipeline()`, make `bind()` mutating - -This is the heart of the public API change. `Pipeline.bind()` is deleted; `PipelineJob.from_pipeline()` is the new way to create a job from a compiled blueprint. - -- [ ] **Step 4.1: Write failing tests** - -Add these tests to `tests/test_pipeline/test_pipeline_job.py` (at the bottom, after existing tests): - -```python -class TestFromPipeline: - """PipelineJob.from_pipeline() creates a runnable job from a compiled Pipeline.""" - - def test_from_pipeline_creates_pipeline_job(self, compiled_pipeline, db): - """from_pipeline returns a PipelineJob with the same topology.""" - from orcapod.pipeline.job import PipelineJob - - job = PipelineJob.from_pipeline(compiled_pipeline, store=db) - assert isinstance(job, PipelineJob) - - def test_from_pipeline_with_sources_binds_them(self, compiled_pipeline, db, source_a): - """Sources passed to from_pipeline are immediately bound.""" - from orcapod.pipeline.job import PipelineJob - - job = PipelineJob.from_pipeline( - compiled_pipeline, store=db, sources={"slot_a": source_a} - ) - assert "slot_a" in job._sources - - def test_pipeline_bind_removed(self, compiled_pipeline): - """Pipeline.bind() no longer exists.""" - assert not hasattr(compiled_pipeline, "bind"), ( - "Pipeline.bind() must be removed — use PipelineJob.from_pipeline() instead" - ) - - -class TestMutatingBind: - """PipelineJob.bind() mutates in place and returns None.""" - - def test_bind_returns_none(self, pipeline_job_with_pipeline, source_a): - result = pipeline_job_with_pipeline.bind(sources={"slot_a": source_a}) - assert result is None - - def test_bind_mutates_sources(self, pipeline_job_with_pipeline, source_a): - pipeline_job_with_pipeline.bind(sources={"slot_a": source_a}) - assert "slot_a" in pipeline_job_with_pipeline._sources - - def test_bind_mutates_store(self, pipeline_job_with_pipeline, db): - pipeline_job_with_pipeline.bind(store=db) - assert pipeline_job_with_pipeline._store is db - - -class TestAsPipeline: - """PipelineJob.as_pipeline() returns a lightweight Pipeline.""" - - def test_as_pipeline_returns_pipeline(self, pipeline_job_with_sources_and_store): - from orcapod.pipeline.graph import Pipeline - - pipeline = pipeline_job_with_sources_and_store.as_pipeline() - assert isinstance(pipeline, Pipeline) - - def test_as_pipeline_node_hashes_match(self, pipeline_job_with_sources_and_store): - """as_pipeline() nodes have matching pipeline_hash to job nodes.""" - job = pipeline_job_with_sources_and_store - pipeline = job.as_pipeline() - - for node_hash in job._persistent_node_map: - assert node_hash in pipeline._persistent_node_map -``` - -> **Note:** The fixture names above (`compiled_pipeline`, `db`, `source_a`, `pipeline_job_with_pipeline`, `pipeline_job_with_sources_and_store`) must be defined in `tests/test_pipeline/conftest.py` or at the top of the test file. Add them as needed based on what already exists. - -- [ ] **Step 4.2: Check existing fixtures in test files** - -```bash -grep -n "^def compiled_pipeline\|^def pipeline_job_with_pipeline\|^def source_a\|^@pytest.fixture" \ - tests/test_pipeline/test_pipeline_job.py | head -20 -``` - -Identify what fixtures already exist and add only the missing ones. - -- [ ] **Step 4.3: Remove Pipeline.bind()** - -In `src/orcapod/pipeline/graph.py`, delete the entire `bind()` method (lines ~313–340 currently): - -```python -# DELETE this entire block: -def bind( - self, - sources: "dict[str, cp.StreamProtocol] | None" = None, - store: "dbp.ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, -) -> "PipelineJob": - ... -``` - -Also remove the `TYPE_CHECKING` import of `PipelineJob` from graph.py if it was only used by `bind()`. - -- [ ] **Step 4.4: Add PipelineJob.from_pipeline() classmethod** - -In `src/orcapod/pipeline/job.py`, add the following classmethod to `PipelineJob`: - -```python -@classmethod -def from_pipeline( - cls, - pipeline: "Pipeline", - store: "ArrowDatabaseProtocol | None" = None, - sources: "dict[str, cp.StreamProtocol] | None" = None, - execution_context: "ExecutionContext | None" = None, -) -> "PipelineJob": - """Create a runnable ``PipelineJob`` from a compiled ``Pipeline``. - - Walks the pipeline's ``_persistent_node_map`` topologically and - creates corresponding ``JobNode`` variants: - - * ``SourceNode`` → ``SourceJobNode(name, schemas, concrete=sources.get(name))`` - * ``FunctionNode`` → ``FunctionJobNode(function_pod, upstream_job_node, label)`` - * ``OperatorNode`` → ``OperatorJobNode(operator, upstream_job_nodes, label)`` - - If *store* is set, ``_distribute_databases()`` is called immediately - so that all ``FunctionJobNode`` / ``OperatorJobNode`` objects have live - DB references before the first ``run()``. - - Args: - pipeline: A compiled ``Pipeline`` (``pipeline._compiled`` must be - ``True``). - store: Database for result caching and operator records. - sources: Mapping of ``SourceNode.name`` → concrete source. - execution_context: Optional execution configuration. - - Returns: - A new ``PipelineJob`` ready to run (or ``bind()`` further). - - Raises: - ValueError: If *pipeline* has not been compiled. - """ - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - from orcapod.utils.lazy_module import LazyModule - - nx = LazyModule("networkx") - - if not pipeline._compiled: - raise ValueError( - "Pipeline must be compiled before creating a PipelineJob from it. " - "Call pipeline.compile() or use auto_compile=True." - ) - - bound_sources: dict[str, cp.StreamProtocol] = dict(sources or {}) - - # Build a topological ordering of the persistent node map - G = pipeline._hash_graph - job_node_map: dict[str, object] = {} # content_hash_str → JobNode - - import networkx as _nx - - for node_hash in _nx.topological_sort(G): - if node_hash not in pipeline._persistent_node_map: - continue - - node = pipeline._persistent_node_map[node_hash] - - if isinstance(node, SourceNode): - concrete = bound_sources.get(node.name) - job_node = SourceJobNode( - name=node.name, - tag_schema=node.tag_schema, - data_schema=node.data_schema, - concrete=concrete, - ) - - elif isinstance(node, FunctionNode): - # Rewire input to the already-built SourceJobNode / FunctionJobNode - original_input_hash = node._input_stream.content_hash().to_string() - upstream_job_node = job_node_map[original_input_hash] - job_node = FunctionJobNode( - function_pod=node._function_pod, - input_stream=upstream_job_node, # type: ignore[arg-type] - label=node._label, - table_scope=node._table_scope, - ) - - elif isinstance(node, OperatorNode): - # Rewire all inputs to already-built job nodes - upstream_job_nodes = tuple( - job_node_map[s.content_hash().to_string()] - for s in node._input_streams - ) - job_node = OperatorJobNode( - operator=node._operator, - input_streams=upstream_job_nodes, # type: ignore[arg-type] - label=node._label, - cache_mode=node._cache_mode, - table_scope=node._table_scope, - ) - - else: - raise TypeError( - f"Unknown node type in pipeline._persistent_node_map: {type(node)}" - ) - - job_node_map[node_hash] = job_node - - # Construct the PipelineJob and inject the built node map - job = cls.__new__(cls) - super(PipelineJob, job).__init__() - job._store = store - job._execution_context = execution_context - job._sources = bound_sources - job._pipeline_name = pipeline._name - job._unresolved_specs = [] - job._has_run = False - job._run_id = None - job._rec_graph_edges = [] - job._rec_upstreams = {} - job._rec_node_lut = {} - job._spec_by_name = {} - - # Copy pipeline graph structure - job._compiled_pipeline = pipeline - job._persistent_node_map = job_node_map # type: ignore[assignment] - job._nodes = {} # populated below - - # Build label → node map by copying from pipeline._nodes - for label, node in pipeline._nodes.items(): - node_hash = node.content_hash().to_string() - if node_hash in job_node_map: - job._nodes[label] = job_node_map[node_hash] # type: ignore[assignment] - - # Wire databases if store is provided - if store is not None: - job._distribute_databases() - - return job -``` - -- [ ] **Step 4.5: Make PipelineJob.bind() mutating (returns None)** - -Replace the existing `bind()` method in `src/orcapod/pipeline/job.py`: - -```python -def bind( - self, - sources: "dict[str, cp.StreamProtocol] | None" = None, - store: "ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, -) -> None: - """Update bindings in place. - - Mutating — modifies ``self`` directly. Existing bindings not mentioned - in this call are preserved. - - When *sources* is provided, each concrete source is validated against - its matching ``SourceNode`` slot schema, then the corresponding - ``SourceJobNode._concrete`` is updated in-place (downstream - ``FunctionJobNode`` objects that hold a reference to the same - ``SourceJobNode`` object automatically see the new concrete without - any cascading reference updates). - - When *store* is provided and differs from the current store, - ``_distribute_databases()`` is called so that all job nodes receive - live DB references immediately. - - Args: - sources: Mapping of ``SourceNode.name`` → concrete source. - store: Replaces the current store and triggers DB redistribution. - execution_context: Replaces the current execution context. - - Raises: - SourceSpecMismatchError: If any source's schema is incompatible. - ValueError: If a source key has no matching ``SourceNode`` slot. - """ - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - - store_changed = store is not None and store is not self._store - - if store is not None: - self._store = store - - if sources is not None: - pipeline = self._compiled_pipeline - if pipeline is not None: - spec_names = { - node.name - for node in pipeline._persistent_node_map.values() - if isinstance(node, SourceNode) - } - unknown = set(sources.keys()) - spec_names - if unknown: - raise ValueError( - f"bind() received source keys with no matching SourceNode: " - f"{sorted(unknown)}. Known names: {sorted(spec_names)}" - ) - # Validate schema for each supplied source - for node in pipeline._persistent_node_map.values(): - if isinstance(node, SourceNode) and node.name in sources: - node.validate(sources[node.name]) - - # Update SourceJobNode._concrete in-place - for job_node in (self._persistent_node_map or {}).values(): - if isinstance(job_node, SourceJobNode) and job_node.name in sources: - job_node._concrete = sources[job_node.name] - # Clear cached hashes so content_hash() reflects new concrete - job_node._content_hash_cache.clear() - - self._sources.update(sources) - - if execution_context is not None: - self._execution_context = execution_context - - if store_changed: - self._distribute_databases() -``` - -- [ ] **Step 4.6: Add _distribute_databases() to PipelineJob** - -Check whether `_distribute_databases()` already exists in `job.py`. If not (it may be in `_build_execution_graph`), add it: - -```python -def _distribute_databases(self) -> None: - """Wire live DB references to all FunctionJobNode and OperatorJobNode objects. - - Called by ``bind()`` when *store* is changed and by ``from_pipeline()`` - when *store* is provided at construction time. - - Raises: - RuntimeError: If ``_store`` is not set. - """ - from orcapod.core.nodes.function_node import FunctionJobNode - from orcapod.core.nodes.operator_node import OperatorJobNode - - if self._store is None: - raise RuntimeError( - "Cannot distribute databases: no store is set. " - "Call bind(store=...) or from_pipeline(..., store=...) first." - ) - - pipeline_db = self._store.at(*self._pipeline_name) - result_db = pipeline_db.at("_result") - - for node in (self._persistent_node_map or {}).values(): - if isinstance(node, FunctionJobNode): - node.attach_databases( - pipeline_database=pipeline_db, - result_database=result_db, - ) - elif isinstance(node, OperatorJobNode): - node.attach_databases(pipeline_database=pipeline_db) -``` - -- [ ] **Step 4.7: Add as_pipeline() to PipelineJob** - -```python -def as_pipeline(self) -> "Pipeline": - """Return the lightweight ``Pipeline`` blueprint for this job. - - Walks ``_persistent_node_map`` and calls ``.as_node()`` on each - ``JobNode`` to obtain the corresponding lightweight ``Node``. - Upstream references in the returned ``Pipeline`` point at the - lightweight nodes, not the ``JobNode`` objects. - - Returns: - A compiled ``Pipeline`` whose ``_persistent_node_map`` contains - only lightweight ``SourceNode`` / ``FunctionNode`` / ``OperatorNode`` - objects with identical ``content_hash()`` and ``pipeline_hash()`` - values to their ``JobNode`` counterparts. - """ - from orcapod.core.nodes.function_node import FunctionJobNode - from orcapod.core.nodes.operator_node import OperatorJobNode - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.pipeline.graph import Pipeline - - import networkx as _nx - - if self._compiled_pipeline is None: - raise RuntimeError( - "PipelineJob has no compiled pipeline. " - "Either use 'with job:' to record a DAG, " - "or create the job via PipelineJob.from_pipeline()." - ) - - G = self._compiled_pipeline._hash_graph - node_map: dict[str, object] = {} - - for node_hash in _nx.topological_sort(G): - if node_hash not in (self._persistent_node_map or {}): - continue - job_node = self._persistent_node_map[node_hash] # type: ignore[index] - node_map[node_hash] = job_node.as_node() - - pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) - pipeline._graph_edges = list(self._compiled_pipeline._graph_edges) - pipeline._upstreams = dict(self._compiled_pipeline._upstreams) - pipeline._node_lut = dict(self._compiled_pipeline._node_lut) - pipeline._hash_graph = self._compiled_pipeline._hash_graph - pipeline._persistent_node_map = node_map # type: ignore[assignment] - pipeline._nodes = { - label: node_map[node.content_hash().to_string()] # type: ignore[index] - for label, node in self._compiled_pipeline._nodes.items() - if node.content_hash().to_string() in node_map - } - pipeline._compiled = True - - return pipeline -``` - -- [ ] **Step 4.8: Run new and full test suite** - -```bash -uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 -``` - -Fix any test that calls `pipeline.bind(...)` — replace with: - -```python -# OLD: -job = pipeline.bind(sources={"slot_a": src}, store=db) - -# NEW: -from orcapod.pipeline.job import PipelineJob -job = PipelineJob.from_pipeline(pipeline, store=db, sources={"slot_a": src}) -``` - -- [ ] **Step 4.9: Commit** - -```bash -git add src/orcapod/pipeline/graph.py \ - src/orcapod/pipeline/job.py \ - tests/test_pipeline/test_pipeline_job.py -git commit -m "refactor(pipeline): remove Pipeline.bind(); add PipelineJob.from_pipeline(), as_pipeline(), mutating bind()" -``` - ---- - -## Task 5: AbstractPipelineBase in pipeline/base.py - -**Files:** -- Create: `src/orcapod/pipeline/base.py` -- Modify: `src/orcapod/pipeline/graph.py` — inherit from `AbstractPipelineBase` -- Modify: `src/orcapod/pipeline/job.py` — inherit from `AbstractPipelineBase` - -This task extracts the shared recording machinery (`_node_lut`, `_upstreams`, `_graph_edges`, `_hash_graph`, `reset()`, `__exit__`, `__getattr__`, `graph` property, etc.) into a common base. - -- [ ] **Step 5.1: Create src/orcapod/pipeline/base.py** - -```python -"""AbstractPipelineBase — shared recording mechanism for Pipeline and PipelineJob.""" -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any - -from orcapod.core.tracker import AutoRegisteringContextBasedTracker -from orcapod.protocols import core_protocols as cp -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import networkx as nx - from orcapod.core.nodes import GraphNode - -else: - nx = LazyModule("networkx") - -logger = logging.getLogger(__name__) - - -class AbstractPipelineBase(AutoRegisteringContextBasedTracker, ABC): - """Shared recording mechanism and graph state for Pipeline and PipelineJob. - - Manages the ``with``-block recording phase: accumulating graph edges, - node LUT entries, and upstream stream references. Subclasses specialise - which node types are created (blueprint vs. job nodes). - - Args: - name: Pipeline name (string or tuple). Used to scope database paths. - tracker_manager: Optional tracker manager override. - """ - - def __init__( - self, - name: str | tuple[str, ...] = "pipeline", - tracker_manager: cp.TrackerManagerProtocol | None = None, - ) -> None: - super().__init__(tracker_manager=tracker_manager) - self._name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) - self._node_lut: dict[str, Any] = {} - self._upstreams: dict[str, cp.StreamProtocol] = {} - self._graph_edges: list[tuple[str, str]] = [] - self._hash_graph: nx.DiGraph = nx.DiGraph() - self._persistent_node_map: dict[str, Any] = {} - self._nodes: dict[str, Any] = {} - self._node_graph: nx.DiGraph | None = None - self._compiled: bool = False - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def name(self) -> tuple[str, ...]: - """Pipeline name tuple.""" - return self._name - - @property - def graph(self) -> nx.DiGraph: - """Directed hash graph of accumulated pipeline structure.""" - return self._hash_graph - - @property - def compiled_nodes(self) -> dict[str, Any]: - """Copy of the compiled nodes dict (label → node).""" - return self._nodes.copy() - - # ------------------------------------------------------------------ - # Recording helpers - # ------------------------------------------------------------------ - - def reset(self) -> None: - """Clear session-scoped recorded state (node LUT, upstreams, edge list). - - Note: - ``_hash_graph`` and ``_persistent_node_map`` are intentionally - *not* cleared — they accumulate across ``with`` blocks. - """ - self._node_lut.clear() - self._upstreams.clear() - self._graph_edges.clear() - - def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: - super().__exit__(exc_type, exc_value, traceback) - if exc_type is None: - self.compile() - - def __getattr__(self, item: str) -> Any: - """Look up compiled nodes by label as attribute access.""" - if item.startswith("_"): - raise AttributeError(item) - nodes = object.__getattribute__(self, "_nodes") - if item in nodes: - return nodes[item] - raise AttributeError( - f"{type(self).__name__!r} has no attribute {item!r}. " - f"Available node labels: {sorted(nodes.keys())}" - ) - - # ------------------------------------------------------------------ - # Abstract — specialised per subclass - # ------------------------------------------------------------------ - - @abstractmethod - def record_function_pod_invocation( - self, - pod: cp.FunctionPodProtocol, - input_stream: cp.StreamProtocol, - label: str | None = None, - ) -> None: - """Record a function pod invocation into the graph.""" - ... - - @abstractmethod - def record_operator_pod_invocation( - self, - pod: cp.OperatorPodProtocol, - upstreams: tuple[cp.StreamProtocol, ...] = (), - label: str | None = None, - ) -> None: - """Record an operator pod invocation into the graph.""" - ... - - @abstractmethod - def compile(self) -> None: - """Compile recorded invocations into a frozen DAG.""" - ... -``` - -- [ ] **Step 5.2: Update Pipeline to inherit from AbstractPipelineBase** - -In `src/orcapod/pipeline/graph.py`: - -1. Add import: `from orcapod.pipeline.base import AbstractPipelineBase` -2. Change class definition: `class Pipeline(AbstractPipelineBase):` (remove `AutoRegisteringContextBasedTracker` from the inheritance since it's now in the base) -3. Remove from `Pipeline.__init__` the attributes already in `AbstractPipelineBase` (`_node_lut`, `_upstreams`, `_graph_edges`, `_hash_graph`, `_name`, `_nodes`, `_persistent_node_map`, `_node_graph`, `_compiled`) -4. Update `super().__init__` call to pass `name=name` and `tracker_manager=tracker_manager` -5. Remove the `reset()` method (now in base), `graph` property (now in base), `compiled_nodes` property (now in base), and `__exit__` if it just calls `compile()` (now in base handles this) -6. Keep `name` property if it does something different, otherwise remove it (base has it) -7. Keep `_auto_compile` flag in Pipeline (base doesn't have it) - -Verify `__exit__` in Pipeline: the base calls `compile()` unconditionally; Pipeline uses `_auto_compile`. Update base's `__exit__` to be abstract (not calling `compile()`), or handle `_auto_compile` differently: - -Actually, keep Pipeline's `__exit__` override: - -```python -def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: - super(AutoRegisteringContextBasedTracker, self).__exit__(exc_type, exc_value, traceback) - if exc_type is None and self._auto_compile: - self.compile() -``` - -Or just keep `_auto_compile` handling by overriding `__exit__` in `Pipeline`: - -```python -def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: - # Skip AbstractPipelineBase.__exit__ (which calls compile() unconditionally) - # and call compile() only if auto_compile is True. - AutoRegisteringContextBasedTracker.__exit__(self, exc_type, exc_value, traceback) - if exc_type is None and self._auto_compile: - self.compile() -``` - -- [ ] **Step 5.3: Update PipelineJob to inherit from AbstractPipelineBase** - -In `src/orcapod/pipeline/job.py`: - -1. Add import: `from orcapod.pipeline.base import AbstractPipelineBase` -2. Change class definition: `class PipelineJob(AbstractPipelineBase):` -3. Update `__init__` to call `super().__init__(name=name, tracker_manager=tracker_manager)` and remove duplicate state initialization - -- [ ] **Step 5.4: Run full test suite** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: All tests pass. - -- [ ] **Step 5.5: Commit** - -```bash -git add src/orcapod/pipeline/base.py \ - src/orcapod/pipeline/graph.py \ - src/orcapod/pipeline/job.py -git commit -m "refactor(pipeline): extract AbstractPipelineBase with shared recording mechanism" -``` - ---- - -## Task 6: Update PipelineJob to use FunctionJobNode / OperatorJobNode during recording - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` - -Currently `PipelineJob.record_function_pod_invocation` creates a `FunctionNode`. After this task it creates a `FunctionJobNode` (DB-ready). Same for operators. PipelineJob's `compile()` then builds `SourceJobNode` leaves and wires job nodes together. - -- [ ] **Step 6.1: Write failing test** - -Add to `tests/test_pipeline/test_pipeline_job.py`: - -```python -class TestPipelineJobUsesJobNodes: - """PipelineJob._persistent_node_map must contain only JobNode variants.""" - - def test_persistent_map_contains_source_job_nodes(self, pipeline_job_with_sources): - from orcapod.core.nodes.source_node import SourceJobNode - - for node in pipeline_job_with_sources._persistent_node_map.values(): - from orcapod.core.nodes.source_node import SourceNodeBase - if isinstance(node, SourceNodeBase): - assert isinstance(node, SourceJobNode), ( - f"Expected SourceJobNode, got {type(node).__name__}" - ) - - def test_persistent_map_contains_function_job_nodes(self, pipeline_job_with_sources): - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode - - for node in pipeline_job_with_sources._persistent_node_map.values(): - if node.node_type == "function": - assert isinstance(node, FunctionJobNode), ( - f"Expected FunctionJobNode, got {type(node).__name__}" - ) - # Thin FunctionNode should NOT appear in PipelineJob - assert type(node) is not FunctionNode, ( - "PipelineJob should use FunctionJobNode, not thin FunctionNode" - ) -``` - -- [ ] **Step 6.2: Run test to confirm failure** - -```bash -uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobUsesJobNodes -v 2>&1 | tail -20 -``` - -Expected: FAIL (currently creates `FunctionNode` in `_persistent_node_map`) - -- [ ] **Step 6.3: Update PipelineJob recording to use FunctionJobNode/OperatorJobNode** - -In `src/orcapod/pipeline/job.py`, update `record_function_pod_invocation`: - -```python -def record_function_pod_invocation( - self, - pod: cp.FunctionPodProtocol, - input_stream: cp.StreamProtocol, - label: str | None = None, -) -> None: - from orcapod.core.nodes.function_node import FunctionJobNode - - input_stream = self._to_node_stream(input_stream) - input_hash = input_stream.content_hash().to_string() - node = FunctionJobNode(function_pod=pod, input_stream=input_stream, label=label) - fn_hash = node.content_hash().to_string() - self._rec_node_lut[fn_hash] = node - self._rec_upstreams[input_hash] = input_stream - self._rec_graph_edges.append((input_hash, fn_hash)) -``` - -Update `record_operator_pod_invocation`: - -```python -def record_operator_pod_invocation( - self, - pod: cp.OperatorPodProtocol, - upstreams: tuple[cp.StreamProtocol, ...] = (), - label: str | None = None, -) -> None: - from orcapod.core.nodes.operator_node import OperatorJobNode - - processed = tuple(self._to_node_stream(s) for s in upstreams) - node = OperatorJobNode(operator=pod, input_streams=processed, label=label) - op_hash = node.content_hash().to_string() - self._rec_node_lut[op_hash] = node - for upstream in processed: - up_hash = upstream.content_hash().to_string() - self._rec_upstreams[up_hash] = upstream - self._rec_graph_edges.append((up_hash, op_hash)) -``` - -- [ ] **Step 6.4: Update PipelineJob._compile_from_recording() to create SourceJobNode leaves** - -In `src/orcapod/pipeline/job.py`, update `_compile_from_recording`: - -```python -def _compile_from_recording(self) -> None: - """Compile recorded edges + node LUT into a pure Pipeline + job node map.""" - from orcapod.pipeline.graph import Pipeline - - # Build the pure Pipeline (SourceNode leaves, thin FunctionNode/OperatorNode) - pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) - pipeline._graph_edges = list(self._rec_graph_edges) - pipeline._upstreams = dict(self._rec_upstreams) - # Convert FunctionJobNode → FunctionNode and OperatorJobNode → OperatorNode - # for the blueprint pipeline - pipeline._node_lut = { - h: node.as_node() for h, node in self._rec_node_lut.items() - } - for edge in self._rec_graph_edges: - pipeline._hash_graph.add_edge(*edge) - for node_hash, node in self._rec_node_lut.items(): - if node_hash in pipeline._hash_graph.nodes: - pipeline._hash_graph.nodes[node_hash]["node_type"] = node.node_type - if node._label: - pipeline._hash_graph.nodes[node_hash]["label"] = node._label - for node_hash, stream in self._rec_upstreams.items(): - if node_hash in pipeline._hash_graph.nodes: - if not pipeline._hash_graph.nodes[node_hash].get("node_type"): - pipeline._hash_graph.nodes[node_hash]["node_type"] = "source" - pipeline.compile() - self._compiled_pipeline = pipeline - - # Build PipelineJob's own job node map using SourceJobNode for leaves - import networkx as _nx - - job_node_map: dict[str, object] = {} - G = pipeline._hash_graph - - for node_hash in _nx.topological_sort(G): - if node_hash not in pipeline._persistent_node_map: - continue - bp_node = pipeline._persistent_node_map[node_hash] - - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode - - if isinstance(bp_node, SourceNode): - concrete = self._sources.get(bp_node.name) - job_node = SourceJobNode( - name=bp_node.name, - tag_schema=bp_node.tag_schema, - data_schema=bp_node.data_schema, - concrete=concrete, - ) - - elif isinstance(bp_node, FunctionNode): - original_input_hash = bp_node._input_stream.content_hash().to_string() - upstream_job_node = job_node_map[original_input_hash] - # Get the original FunctionJobNode from rec_node_lut - rec_node = self._rec_node_lut.get(node_hash) - table_scope = rec_node._table_scope if rec_node is not None else "pipeline_hash" - job_node = FunctionJobNode( - function_pod=bp_node._function_pod, - input_stream=upstream_job_node, # type: ignore[arg-type] - label=bp_node._label, - table_scope=table_scope, - ) - - elif isinstance(bp_node, OperatorNode): - upstream_job_nodes = tuple( - job_node_map[s.content_hash().to_string()] - for s in bp_node._input_streams - ) - rec_node = self._rec_node_lut.get(node_hash) - table_scope = rec_node._table_scope if rec_node is not None else "pipeline_hash" - cache_mode = rec_node._cache_mode if rec_node is not None else None - from orcapod.types import CacheMode - job_node = OperatorJobNode( - operator=bp_node._operator, - input_streams=upstream_job_nodes, # type: ignore[arg-type] - label=bp_node._label, - cache_mode=cache_mode or CacheMode.OFF, - table_scope=table_scope, - ) - - else: - raise TypeError( - f"Unknown blueprint node type in pipeline._persistent_node_map: {type(bp_node)}" - ) - - job_node_map[node_hash] = job_node - - self._persistent_node_map = job_node_map # type: ignore[assignment] - - # Build label → job node map - self._nodes = { - label: job_node_map[node.content_hash().to_string()] # type: ignore[index] - for label, node in pipeline._nodes.items() - if node.content_hash().to_string() in job_node_map - } - - # Wire databases if store is set - if self._store is not None: - self._distribute_databases() -``` - -- [ ] **Step 6.5: Run test suite** - -```bash -uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 -``` - -Fix any failures. Common issues: -- Tests that access `job._compiled_pipeline._persistent_node_map` expecting `FunctionNode` — now it has `FunctionNode` (blueprint), but `job._persistent_node_map` has `FunctionJobNode`. Update assertions. -- Tests calling `_build_execution_graph()` — check if that method still works or needs updating. - -- [ ] **Step 6.6: Commit** - -```bash -git add src/orcapod/pipeline/job.py -git commit -m "refactor(pipeline): PipelineJob recording now creates FunctionJobNode/OperatorJobNode/SourceJobNode" -``` - ---- - -## Task 7: Update serialization for the new node format - -**Files:** -- Modify: `src/orcapod/pipeline/serialization.py` -- Modify: `src/orcapod/pipeline/graph.py` (save/load methods) -- Modify: `tests/test_pipeline/test_serialization.py` - -The key change: serialized source nodes no longer wrap a `SourceSpec` — they are plain `SourceNode` objects with `source_type: "node"`. Format versions bump. - -- [ ] **Step 7.1: Write failing serialization tests** - -Add to `tests/test_pipeline/test_serialization.py`: - -```python -class TestNewSerializationFormat: - def test_save_load_roundtrip_with_source_node(self, tmp_path, compiled_pipeline): - """Pipeline.save/load round-trip preserves SourceNode slots.""" - from orcapod.core.nodes.source_node import SourceNode - from orcapod.pipeline.graph import Pipeline - - save_path = tmp_path / "test_pipeline.json" - compiled_pipeline.save(save_path) - - loaded = Pipeline.load(save_path) - assert loaded._compiled - - for node in loaded._persistent_node_map.values(): - if node.node_type == "source": - assert isinstance(node, SourceNode) - - def test_saved_format_has_source_node_type(self, tmp_path, compiled_pipeline): - import json - - save_path = tmp_path / "test_pipeline.json" - compiled_pipeline.save(save_path) - - with open(save_path) as f: - data = json.load(f) - - for node_data in data["nodes"].values(): - if node_data["node_type"] == "source": - assert node_data.get("source_config", {}).get("source_type") == "node", ( - "Source nodes should be serialized with source_type='node'" - ) - - def test_format_version_is_0_3(self, tmp_path, compiled_pipeline): - import json - - save_path = tmp_path / "test_pipeline.json" - compiled_pipeline.save(save_path) - - with open(save_path) as f: - data = json.load(f) - - assert data["orcapod_pipeline_version"] == "0.3" -``` - -- [ ] **Step 7.2: Run tests to confirm failure** - -```bash -uv run pytest tests/test_pipeline/test_serialization.py::TestNewSerializationFormat -v 2>&1 | tail -20 -``` - -- [ ] **Step 7.3: Update Pipeline.save() for new SourceNode format** - -In `src/orcapod/pipeline/graph.py`, find `save()` and update the source-node serialization block. Find the block that serializes source nodes (currently writes `SourceSpec` fields): - -```python -# OLD (inside save() — look for the SourceNode handling block) -from orcapod.core.sources.source_spec import SourceSpec -if isinstance(node.stream, SourceSpec): - spec = node.stream - nodes[node_hash] = { - "node_type": "source", - "label": node.label, - "source_config": { - "source_type": "spec", - "name": spec.name, - "tag_schema": serialize_schema(spec.tag_schema), - "data_schema": serialize_schema(spec.data_schema), - }, - } -``` - -Replace with: - -```python -# NEW — SourceNode is the leaf directly -from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass -if isinstance(node, SourceNodeClass): - nodes[node_hash] = { - "node_type": "source", - "label": node.label, - "source_config": { - "source_type": "node", - "name": node.name, - "tag_schema": serialize_schema(node.tag_schema), - "data_schema": serialize_schema(node.data_schema), - }, - } -``` - -Update the format version constant in `src/orcapod/pipeline/serialization.py`: - -```python -# OLD: -PIPELINE_FORMAT_VERSION = "0.2" -# NEW: -PIPELINE_FORMAT_VERSION = "0.3" -``` - -- [ ] **Step 7.4: Update Pipeline.load() to reconstruct SourceNode from new format** - -In `src/orcapod/pipeline/graph.py`, find `load()` and update source node reconstruction: - -```python -# NEW — inside load(), source node handling -if node_data["node_type"] == "source": - source_config = node_data.get("source_config", {}) - source_type = source_config.get("source_type") - if source_type == "node": - from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass - from orcapod.pipeline.serialization import deserialize_schema - node = SourceNodeClass( - name=source_config["name"], - tag_schema=deserialize_schema(source_config["tag_schema"]), - data_schema=deserialize_schema(source_config["data_schema"]), - ) - elif source_type == "spec": - # Backward-compat: load old format (v0.2) that used SourceSpec - from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass - from orcapod.pipeline.serialization import deserialize_schema - node = SourceNodeClass( - name=source_config["name"], - tag_schema=deserialize_schema(source_config["tag_schema"]), - data_schema=deserialize_schema(source_config["data_schema"]), - ) - else: - raise ValueError( - f"Unknown source_type {source_type!r} in pipeline descriptor." - ) -``` - -> **Note:** For backward compatibility with v0.2 format, `source_type == "spec"` reconstructs a `SourceNode` with the same name/schemas — identical hashes are preserved since `SourceNode.identity_structure()` matches old `SourceSpec.identity_structure()`. - -- [ ] **Step 7.5: Run serialization tests** - -```bash -uv run pytest tests/test_pipeline/test_serialization.py -v --tb=short 2>&1 | tail -40 -``` - -Expected: All tests pass (including the new ones and old ones). - -- [ ] **Step 7.6: Commit** - -```bash -git add src/orcapod/pipeline/graph.py \ - src/orcapod/pipeline/serialization.py \ - tests/test_pipeline/test_serialization.py -git commit -m "refactor(serialization): update Pipeline save/load for SourceNode format; bump to v0.3" -``` - ---- - -## Task 8: Delete SourceSpec and clean up all references - -**Files:** -- Delete: `src/orcapod/core/sources/source_spec.py` -- Delete: `tests/test_core/sources/test_source_spec.py` -- Modify: `src/orcapod/core/sources/__init__.py` -- Modify: `src/orcapod/__init__.py` -- Modify: `src/orcapod/errors.py` — rename `SourceSpecMismatchError` → keep as alias -- Modify: all remaining test files that reference `SourceSpec` - -- [ ] **Step 8.1: Find all remaining SourceSpec references** - -```bash -grep -rn "SourceSpec\|source_spec" src/ tests/ --include="*.py" | grep -v "\.pyc" -``` - -Record every file that still imports or references `SourceSpec`. - -- [ ] **Step 8.2: Update src/orcapod/__init__.py** - -Replace `SourceSpec` export with `SourceNode`: - -```python -# OLD: -from .core.sources.source_spec import SourceSpec - -# NEW: -from .core.nodes.source_node import SourceNode -``` - -- [ ] **Step 8.3: Update src/orcapod/core/sources/__init__.py** - -Remove `SourceSpec` from the exports. If the file only exported `SourceSpec`, the file can be left with just the remaining exports (or emptied). - -- [ ] **Step 8.4: Update errors.py — rename SourceSpecMismatchError** - -`SourceSpecMismatchError` is the right name since it describes a schema mismatch on a source node slot. Keep the name but update the docstring: - -```python -class SourceSpecMismatchError(ValueError): - """Raised when a concrete source's schema is incompatible with a SourceNode slot. - - Previously named in terms of ``SourceSpec``; the error class name is preserved - for compatibility with any code that catches it by name. - """ -``` - -- [ ] **Step 8.5: Update remaining SourceSpec references in test files** - -For each file from Step 8.1 that still imports `SourceSpec`: - -**`tests/test_pipeline/test_pipeline.py`** — replace: -```python -# OLD: -from orcapod.core.sources.source_spec import SourceSpec -... -spec_a = SourceSpec(name="a", tag_schema=..., data_schema=...) -with pipeline: - result = my_pod(spec_a) - -# NEW: -from orcapod.core.nodes.source_node import SourceNode -... -spec_a = SourceNode(name="a", tag_schema=..., data_schema=...) -with pipeline: - result = my_pod(spec_a) -``` - -**`tests/test_pipeline/test_pipeline_job.py`** — same pattern. - -**`tests/test_core/test_tracker.py`** — if it uses `SourceSpec`, replace with `SourceNode`. - -Run after each file update: - -```bash -uv run pytest -v --tb=short 2>&1 | tail -20 -``` - -- [ ] **Step 8.6: Delete source_spec.py** - -```bash -git rm src/orcapod/core/sources/source_spec.py -git rm tests/test_core/sources/test_source_spec.py -``` - -- [ ] **Step 8.7: Run full test suite** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: All tests pass, no `SourceSpec` references remaining. - -- [ ] **Step 8.8: Final grep to confirm SourceSpec is gone** - -```bash -grep -rn "SourceSpec\|source_spec" src/ tests/ --include="*.py" -``` - -Expected: No output (zero matches). - -- [ ] **Step 8.9: Commit** - -```bash -git add -A -git commit -m "refactor(sources): delete SourceSpec; update all references to SourceNode (ENG-493)" -``` - ---- - -## Task 9: Integration test sweep and PR - -**Files:** -- Run all tests, fix stragglers -- Create PR - -- [ ] **Step 9.1: Run full test suite including integration tests** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tee /tmp/test_results.txt -grep -E "FAILED|ERROR" /tmp/test_results.txt -``` - -- [ ] **Step 9.2: Fix any remaining failures** - -Common failure patterns at this stage: -- `_build_execution_graph()` in `job.py` may still reference `SourceSpec` — update to `SourceNode` -- Serialization load tests for old v0.2 format — verify backward-compat path works -- Orchestrator tests that build execution graphs — update to use `from_pipeline()` pattern - -For each failure, diagnose root cause, implement fix, re-run the specific test. - -- [ ] **Step 9.3: Verify hash stability end-to-end** - -```bash -uv run python -c " -from orcapod.core.nodes.source_node import SourceNode -from orcapod.core.sources.source_spec import SourceSpec -from orcapod.types import Schema - -tag = Schema({'id': int}) -data = Schema({'value': float}) - -# These MUST be equal for DB path stability -old_hash = SourceSpec(name='x', tag_schema=tag, data_schema=data).content_hash() -new_hash = SourceNode(name='x', tag_schema=tag, data_schema=data).content_hash() - -assert old_hash == new_hash, f'Hash mismatch! old={old_hash}, new={new_hash}' -print('Hash stability: PASS') - -old_pipe = SourceSpec(name='x', tag_schema=tag, data_schema=data).pipeline_hash() -new_pipe = SourceNode(name='x', tag_schema=tag, data_schema=data).pipeline_hash() -assert old_pipe == new_pipe, f'Pipeline hash mismatch!' -print('Pipeline hash stability: PASS') -" -``` - -> **Note:** This check must pass before the PR is created. If SourceSpec has already been deleted at this point, adjust to compare against a recorded reference hash instead. - -- [ ] **Step 9.4: Update src/orcapod/__init__.py exports** - -Ensure `SourceNode` is properly exported (was done in Task 8, verify): - -```python -from .core.nodes.source_node import SourceNode -from .pipeline.job import PipelineJob -from .pipeline.graph import Pipeline -``` - -- [ ] **Step 9.5: Final test run** - -```bash -uv run pytest tests/ --tb=short 2>&1 | tail -10 -``` - -Expected: `X passed, 0 failed, 0 errors` - -- [ ] **Step 9.6: Commit any remaining fixes** - -```bash -git add -A -git commit -m "fix(pipeline): address integration test failures after pure-descriptor refactor" -``` - -- [ ] **Step 9.7: Push and create PR** - -```bash -git push -u origin eywalker/eng-493-refactor-pipeline-into-a-pure-computational-descriptor -``` - -Create PR targeting `dev`: - -```bash -gh pr create \ - --title "refactor(pipeline): Pipeline pure-descriptor refactor — ENG-493" \ - --base dev \ - --body "$(cat <<'EOF' -## Summary - -- `Pipeline` now stores only lightweight blueprint nodes (`FunctionNode`, `OperatorNode`, `SourceNode`) with no DB references -- `PipelineJob` stores DB-backed job nodes (`FunctionJobNode`, `OperatorJobNode`, `SourceJobNode`) -- `SourceSpec` deleted; replaced by `SourceNode` with bit-identical `content_hash()` / `pipeline_hash()` (DB paths preserved) -- `Pipeline.bind()` removed; replaced by `PipelineJob.from_pipeline(pipeline, store=..., sources=...)` -- `PipelineJob.bind()` is now mutating (returns `None`); updates `SourceJobNode._concrete` in-place -- `PipelineJob.as_pipeline()` produces a lightweight `Pipeline` from a job -- `AbstractPipelineBase` extracted to `pipeline/base.py` — shared recording machinery -- Serialization format bumped to v0.3 (backward-compatible load of v0.2) - -Closes ENG-493 - -## Test plan -- [ ] `uv run pytest tests/test_core/nodes/` — new node hierarchy unit tests -- [ ] `uv run pytest tests/test_pipeline/` — full pipeline test suite -- [ ] `uv run pytest tests/` — complete test suite - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - ---- - -## Self-Review Against Spec - -### Spec coverage check - -| Spec requirement | Covered by task | -|---|---| -| `Pipeline._persistent_node_map` contains only lightweight nodes | Tasks 5, 6 | -| `PipelineJob._persistent_node_map` contains only JobNodes | Task 6 | -| `SourceSpec` eliminated | Task 8 | -| `SourceNode` as user-facing input slot | Task 1 | -| `Pipeline.bind()` removed | Task 4 | -| `PipelineJob.from_pipeline()` classmethod | Task 4 | -| `PipelineJob.bind()` mutating | Task 4 | -| `PipelineJob.as_pipeline()` | Task 4 | -| `SourceJobNode._concrete` mutable for in-place update | Task 1 | -| Hash stability: `SourceNode.content_hash() == SourceSpec.content_hash()` | Task 1 (tests) | -| `pipeline_hash()` always schema-based for `SourceJobNode` | Task 1 (tests) | -| `FunctionNode.iter_data()` raises `PipelineJobRequiredError` | Task 2 | -| `FunctionJobNode.as_node()` returns `FunctionNode` | Task 2 | -| `OperatorNode.iter_data()` raises `PipelineJobRequiredError` | Task 3 | -| `OperatorJobNode.as_node()` returns `OperatorNode` | Task 3 | -| `AbstractPipelineBase` in `pipeline/base.py` | Task 5 | -| Serialization format v0.3, backward-compat v0.2 load | Task 7 | -| All tests updated | Tasks 1–8 | - -### Notes on `_build_execution_graph()` - -The current `PipelineJob._build_execution_graph()` creates `FunctionNode`/`OperatorNode` with DB attached from scratch. After Task 6, the `_persistent_node_map` already has `FunctionJobNode`/`OperatorJobNode`. The `_build_execution_graph()` method may need updating or removal — it may be replaced by `_distribute_databases()` + the new job node map. Check what calls it (orchestrators) and update those call sites. - -### `tracker_manager` parameter in AbstractPipelineBase - -`PipelineJob.__init__` currently does not accept `name` or `tracker_manager` via parent `__init__` in the same form. Ensure the `super().__init__` chains are correct after Task 5. diff --git a/superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md b/superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md deleted file mode 100644 index 6fd4e8927..000000000 --- a/superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md +++ /dev/null @@ -1,487 +0,0 @@ -# ENG-493 Task 6: PipelineJob Recording Uses JobNode Types - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** After a `with job:` block, `job._persistent_node_map` contains only `JobNode` variants (`SourceJobNode`, `FunctionJobNode`, `OperatorJobNode`), not blueprint types. - -**Architecture:** `record_function_pod_invocation` and `record_operator_pod_invocation` now create `FunctionJobNode`/`OperatorJobNode` in `_rec_node_lut`. `_compile_from_recording` converts these to lightweight `FunctionNode`/`OperatorNode` (via `.as_node()`) for the blueprint `Pipeline._node_lut`, then builds `job._persistent_node_map` with `SourceJobNode` leaves by walking the compiled pipeline topologically. `_build_execution_graph` already uses `FunctionNode`/`OperatorNode` from the blueprint pipeline and is unchanged. - -**Tech Stack:** Python, uv, pytest, networkx, orcapod nodes hierarchy. - ---- - -## File Map - -| File | Change | -|---|---| -| `src/orcapod/pipeline/job.py` | Modify `record_function_pod_invocation`, `record_operator_pod_invocation`, and `_compile_from_recording` | -| `tests/test_pipeline/test_pipeline_job.py` | Add `TestPipelineJobUsesJobNodes` class with fixture | - ---- - -### Task 1: Write failing tests for job node types in `_persistent_node_map` - -**Files:** -- Modify: `tests/test_pipeline/test_pipeline_job.py` - -- [ ] **Step 1: Add the `pipeline_job_with_sources` fixture and `TestPipelineJobUsesJobNodes` class** - -Open `tests/test_pipeline/test_pipeline_job.py` and add the following immediately before the final `class TestFromPipeline:` block (after the `compiled_pipeline`/`db`/`source_a`/`source_b`/`pipeline_job`/`pipeline_job_complete` fixtures, around line 720): - -```python -@pytest.fixture -def pipeline_job_with_sources(store): - """A PipelineJob created via with-block using concrete sources + a FunctionPod.""" - src_a, src_b = _make_two_sources() - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, src_b, label="joiner") - pod(joined, label="adder") - return job - - -class TestPipelineJobUsesJobNodes: - """PipelineJob._persistent_node_map must contain only JobNode variants after recording.""" - - def test_persistent_map_has_source_job_nodes(self, pipeline_job_with_sources): - """Source entries in PipelineJob._persistent_node_map must be SourceJobNode.""" - from orcapod.core.nodes.source_node import SourceJobNode, SourceNodeBase - - assert pipeline_job_with_sources._persistent_node_map is not None, ( - "_persistent_node_map must be set after with-block" - ) - for node in pipeline_job_with_sources._persistent_node_map.values(): - if isinstance(node, SourceNodeBase): - assert isinstance(node, SourceJobNode), ( - f"Expected SourceJobNode but got {type(node).__name__}" - ) - - def test_persistent_map_has_function_job_nodes(self, pipeline_job_with_sources): - """Function entries in PipelineJob._persistent_node_map must be FunctionJobNode.""" - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNodeBase - - assert pipeline_job_with_sources._persistent_node_map is not None - fn_nodes = [ - n for n in pipeline_job_with_sources._persistent_node_map.values() - if isinstance(n, FunctionNodeBase) - ] - assert len(fn_nodes) >= 1, "Expected at least one FunctionJobNode" - for node in fn_nodes: - assert isinstance(node, FunctionJobNode), ( - f"Expected FunctionJobNode but got {type(node).__name__}" - ) - - def test_persistent_map_has_operator_job_nodes(self, pipeline_job_with_sources): - """Operator entries in PipelineJob._persistent_node_map must be OperatorJobNode.""" - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase - - assert pipeline_job_with_sources._persistent_node_map is not None - op_nodes = [ - n for n in pipeline_job_with_sources._persistent_node_map.values() - if isinstance(n, OperatorNodeBase) - ] - assert len(op_nodes) >= 1, "Expected at least one OperatorJobNode" - for node in op_nodes: - assert isinstance(node, OperatorJobNode), ( - f"Expected OperatorJobNode but got {type(node).__name__}" - ) - - def test_blueprint_pipeline_still_has_lightweight_nodes(self, pipeline_job_with_sources): - """The compiled pipeline's _persistent_node_map still has lightweight nodes.""" - from orcapod.core.nodes.function_node import FunctionJobNode - from orcapod.core.nodes.operator_node import OperatorJobNode - - for node in pipeline_job_with_sources.pipeline._persistent_node_map.values(): - assert not isinstance(node, FunctionJobNode), ( - "Blueprint pipeline must not contain FunctionJobNode" - ) - assert not isinstance(node, OperatorJobNode), ( - "Blueprint pipeline must not contain OperatorJobNode" - ) -``` - -- [ ] **Step 2: Run the new tests to verify they fail** - -```bash -cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ -uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobUsesJobNodes -v 2>&1 | tail -20 -``` - -Expected: FAIL — `_persistent_node_map` is `None` after with-block (returns `AssertionError: _persistent_node_map must be set after with-block`), and/or `FunctionNode`/`OperatorNode` found instead of job variants. - ---- - -### Task 2: Update `record_function_pod_invocation` to create `FunctionJobNode` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` (lines ~197–221) - -- [ ] **Step 1: Replace the body of `record_function_pod_invocation`** - -Find this block in `src/orcapod/pipeline/job.py`: - -```python - def record_function_pod_invocation( - self, - pod: cp.FunctionPodProtocol, - input_stream: cp.StreamProtocol, - label: str | None = None, - ) -> None: - """Record a function pod invocation, promoting concrete sources to specs. - - Args: - pod: The function pod being invoked. - input_stream: The upstream stream (concrete source or spec). - label: Optional label for the resulting node. - """ - from orcapod.core.nodes import FunctionNode - - input_stream = self._to_node_stream(input_stream) - - input_hash = input_stream.content_hash().to_string() - function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) - fn_hash = function_node.content_hash().to_string() - - self._rec_node_lut[fn_hash] = function_node - self._rec_upstreams[input_hash] = input_stream - self._rec_graph_edges.append((input_hash, fn_hash)) -``` - -Replace with: - -```python - def record_function_pod_invocation( - self, - pod: cp.FunctionPodProtocol, - input_stream: cp.StreamProtocol, - label: str | None = None, - ) -> None: - """Record a function pod invocation, promoting concrete sources to specs. - - Args: - pod: The function pod being invoked. - input_stream: The upstream stream (concrete source or spec). - label: Optional label for the resulting node. - """ - from orcapod.core.nodes.function_node import FunctionJobNode - - input_stream = self._to_node_stream(input_stream) - - input_hash = input_stream.content_hash().to_string() - function_node = FunctionJobNode(function_pod=pod, input_stream=input_stream, label=label) - fn_hash = function_node.content_hash().to_string() - - self._rec_node_lut[fn_hash] = function_node - self._rec_upstreams[input_hash] = input_stream - self._rec_graph_edges.append((input_hash, fn_hash)) -``` - -The only change is `FunctionNode` → `FunctionJobNode` in the import and construction. The hash is identical because `FunctionJobNode` inherits `content_hash()` from `FunctionNodeBase` with the same identity structure. - ---- - -### Task 3: Update `record_operator_pod_invocation` to create `OperatorJobNode` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` (lines ~222–246) - -- [ ] **Step 1: Replace the body of `record_operator_pod_invocation`** - -Find this block in `src/orcapod/pipeline/job.py`: - -```python - def record_operator_pod_invocation( - self, - pod: cp.OperatorPodProtocol, - upstreams: tuple[cp.StreamProtocol, ...] = (), - label: str | None = None, - ) -> None: - """Record an operator pod invocation, promoting concrete sources to specs. - - Args: - pod: The operator pod being invoked. - upstreams: Upstream streams (concrete sources or specs). - label: Optional label for the resulting node. - """ - from orcapod.core.nodes import OperatorNode - - processed = tuple(self._to_node_stream(s) for s in upstreams) - - operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) - op_hash = operator_node.content_hash().to_string() - - self._rec_node_lut[op_hash] = operator_node - for upstream in processed: - up_hash = upstream.content_hash().to_string() - self._rec_upstreams[up_hash] = upstream - self._rec_graph_edges.append((up_hash, op_hash)) -``` - -Replace with: - -```python - def record_operator_pod_invocation( - self, - pod: cp.OperatorPodProtocol, - upstreams: tuple[cp.StreamProtocol, ...] = (), - label: str | None = None, - ) -> None: - """Record an operator pod invocation, promoting concrete sources to specs. - - Args: - pod: The operator pod being invoked. - upstreams: Upstream streams (concrete sources or specs). - label: Optional label for the resulting node. - """ - from orcapod.core.nodes.operator_node import OperatorJobNode - - processed = tuple(self._to_node_stream(s) for s in upstreams) - - operator_node = OperatorJobNode(operator=pod, input_streams=processed, label=label) - op_hash = operator_node.content_hash().to_string() - - self._rec_node_lut[op_hash] = operator_node - for upstream in processed: - up_hash = upstream.content_hash().to_string() - self._rec_upstreams[up_hash] = upstream - self._rec_graph_edges.append((up_hash, op_hash)) -``` - -The only change is `OperatorNode` → `OperatorJobNode` in the import and construction. - ---- - -### Task 4: Update `_compile_from_recording` to convert JobNodes for blueprint and build job `_persistent_node_map` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` (lines ~103–130) - -**Key insight:** `Pipeline.compile()` expects `FunctionNode`/`OperatorNode` in `pipeline._node_lut` — it raises `TypeError` for any other type. So `_compile_from_recording` must convert `FunctionJobNode`→`FunctionNode` and `OperatorJobNode`→`OperatorNode` via `.as_node()` before calling `pipeline.compile()`. Then it builds `job._persistent_node_map` by walking the compiled pipeline topologically, creating `SourceJobNode` leaves from the concrete sources captured in `_sources`. - -- [ ] **Step 1: Replace `_compile_from_recording`** - -Find this entire method in `src/orcapod/pipeline/job.py`: - -```python - def _compile_from_recording(self) -> None: - """Compile the recorded edges into a pure Pipeline.""" - from orcapod.pipeline.graph import Pipeline - - pipeline = Pipeline(name=self._name, auto_compile=False) - # Inject the recording state into the pipeline - pipeline._graph_edges = list(self._rec_graph_edges) - pipeline._upstreams = dict(self._rec_upstreams) - pipeline._node_lut = dict(self._rec_node_lut) - # Rebuild hash graph from edges - for edge in self._rec_graph_edges: - pipeline._hash_graph.add_edge(*edge) - - # Annotate node_type on each recorded node (function/operator). - for node_hash, node in self._rec_node_lut.items(): - if node_hash in pipeline._hash_graph.nodes: - pipeline._hash_graph.nodes[node_hash]["node_type"] = node.node_type - if node.label: - pipeline._hash_graph.nodes[node_hash]["label"] = node.label - - # Annotate upstream (source) nodes that are not in _rec_node_lut. - for node_hash, stream in self._rec_upstreams.items(): - if node_hash in pipeline._hash_graph.nodes: - if not pipeline._hash_graph.nodes[node_hash].get("node_type"): - pipeline._hash_graph.nodes[node_hash]["node_type"] = "source" - - pipeline.compile() - self._compiled_pipeline = pipeline -``` - -Replace with: - -```python - def _compile_from_recording(self) -> None: - """Compile the recorded edges into a pure Pipeline and build the job node map. - - ``_rec_node_lut`` now contains ``FunctionJobNode`` / ``OperatorJobNode`` objects - (set by ``record_function_pod_invocation`` / ``record_operator_pod_invocation``). - This method: - - 1. Converts each recorded job node to its lightweight blueprint counterpart via - ``.as_node()`` and injects the result into ``pipeline._node_lut`` so that - ``Pipeline.compile()`` sees only ``FunctionNode`` / ``OperatorNode`` objects. - 2. After compiling the blueprint pipeline, walks it topologically to build - ``self._persistent_node_map`` using ``SourceJobNode`` for leaf nodes - (concrete sources are taken from ``self._sources``) and the original recorded - ``FunctionJobNode`` / ``OperatorJobNode`` objects for non-leaf nodes. - """ - import networkx as _nx - from orcapod.pipeline.graph import Pipeline - from orcapod.core.nodes.source_node import SourceJobNode, SourceNodeBase - from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNodeBase - from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase - - pipeline = Pipeline(name=self._name, auto_compile=False) - # Inject the recording state into the pipeline, converting job nodes → blueprint nodes - pipeline._graph_edges = list(self._rec_graph_edges) - pipeline._upstreams = dict(self._rec_upstreams) - pipeline._node_lut = { - h: node.as_node() for h, node in self._rec_node_lut.items() - } - # Rebuild hash graph from edges - for edge in self._rec_graph_edges: - pipeline._hash_graph.add_edge(*edge) - - # Annotate node_type on each recorded node (function/operator). - for node_hash, node in self._rec_node_lut.items(): - if node_hash in pipeline._hash_graph.nodes: - pipeline._hash_graph.nodes[node_hash]["node_type"] = node.node_type - if node.label: - pipeline._hash_graph.nodes[node_hash]["label"] = node.label - - # Annotate upstream (source) nodes that are not in _rec_node_lut. - for node_hash, stream in self._rec_upstreams.items(): - if node_hash in pipeline._hash_graph.nodes: - if not pipeline._hash_graph.nodes[node_hash].get("node_type"): - pipeline._hash_graph.nodes[node_hash]["node_type"] = "source" - - pipeline.compile() - self._compiled_pipeline = pipeline - - # Build PipelineJob's own job node map walking compiled pipeline topologically. - # Leaf nodes (SourceNode) become SourceJobNode with concrete binding from _sources. - # Non-leaf nodes reuse the FunctionJobNode/OperatorJobNode from _rec_node_lut. - G = pipeline._hash_graph - job_node_map: dict[str, object] = {} - - for node_hash in _nx.topological_sort(G): - if node_hash not in pipeline._persistent_node_map: - continue - - bp_node = pipeline._persistent_node_map[node_hash] - - if isinstance(bp_node, SourceNodeBase): - concrete = self._sources.get(bp_node.name) - job_node: object = SourceJobNode( - name=bp_node.name, - tag_schema=bp_node.tag_schema, - data_schema=bp_node.data_schema, - concrete=concrete, - ) - elif isinstance(bp_node, FunctionNodeBase): - # Reuse the FunctionJobNode from _rec_node_lut, rewired to upstream job node. - # The original rec node has the correct table_scope and tracker_manager. - rec_node = self._rec_node_lut[node_hash] - original_input_hash = bp_node._input_stream.content_hash().to_string() - upstream_job_node = job_node_map[original_input_hash] - job_node = FunctionJobNode( - function_pod=rec_node._function_pod, - input_stream=upstream_job_node, - label=rec_node._label, - table_scope=rec_node._table_scope, - tracker_manager=rec_node.tracker_manager, - ) - elif isinstance(bp_node, OperatorNodeBase): - rec_node = self._rec_node_lut[node_hash] - upstream_job_nodes = tuple( - job_node_map[s.content_hash().to_string()] - for s in bp_node._input_streams - ) - job_node = OperatorJobNode( - operator=rec_node._operator, - input_streams=upstream_job_nodes, - label=rec_node._label, - table_scope=rec_node._table_scope, - tracker_manager=rec_node.tracker_manager, - ) - else: - raise TypeError( - f"Unknown blueprint node type in compiled pipeline: {type(bp_node)}" - ) - - job_node_map[node_hash] = job_node - - self._persistent_node_map = job_node_map - - # Build label → job node map from pipeline._nodes - self._nodes = { - label: job_node_map[node.content_hash().to_string()] - for label, node in pipeline._nodes.items() - if node.content_hash().to_string() in job_node_map - } - - # Wire databases if store is already set - if self._store is not None: - self._distribute_databases() -``` - ---- - -### Task 5: Run the new tests — verify they pass - -- [ ] **Step 1: Run `TestPipelineJobUsesJobNodes`** - -```bash -cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ -uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobUsesJobNodes -v 2>&1 | tail -20 -``` - -Expected: All 4 tests PASS. - -- [ ] **Step 2: Run the full pipeline test suite** - -```bash -cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ -uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -50 -``` - -Expected: All tests pass. Watch for: -- `TestPipelineJobEndToEnd::test_end_to_end_source_join_function` — checks `compiled_nodes["joiner"]` is `OperatorNode` and `compiled_nodes["adder"]` is `FunctionNode`. These look at `job.pipeline.compiled_nodes` (the blueprint pipeline), not `job._persistent_node_map`, so they should still pass. -- `TestPipelineJobRun` — exercises `job.run()` via `_build_execution_graph` which reads from `pipeline._node_lut` (blueprint); should be unaffected. - -If any test fails due to type checks on `_persistent_node_map` expecting `FunctionNode`/`OperatorNode`, update those assertions to check for `FunctionJobNode`/`OperatorJobNode` respectively. - -- [ ] **Step 3: Run the full test suite** - -```bash -cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ -uv run pytest tests/ -v --tb=short 2>&1 | tail -60 -``` - -Expected: All tests pass. - ---- - -### Task 6: Commit - -- [ ] **Step 1: Commit the changes** - -```bash -cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ -git add src/orcapod/pipeline/job.py tests/test_pipeline/test_pipeline_job.py && \ -git commit -m "refactor(pipeline): PipelineJob recording creates FunctionJobNode/OperatorJobNode; compile builds SourceJobNode leaves" -``` - ---- - -## Self-Review - -**Spec coverage:** -- `record_function_pod_invocation` → `FunctionJobNode`: Task 2. ✓ -- `record_operator_pod_invocation` → `OperatorJobNode`: Task 3. ✓ -- `_compile_from_recording` converts via `.as_node()` for blueprint: Task 4. ✓ -- `_compile_from_recording` builds `_persistent_node_map` with `SourceJobNode`: Task 4. ✓ -- `job._nodes` label map updated: Task 4. ✓ -- `_distribute_databases()` called when store set: Task 4. ✓ -- Blueprint pipeline still has lightweight nodes: tested in Task 1, step 1 (4th test). ✓ -- Existing tests not broken: Task 5 full suite run. ✓ - -**Placeholder scan:** No TBDs, TODOs, or vague steps found. - -**Type consistency:** -- `FunctionJobNode` referenced in Tasks 2, 4 — both use `from orcapod.core.nodes.function_node import FunctionJobNode`. ✓ -- `OperatorJobNode` referenced in Tasks 3, 4 — both use `from orcapod.core.nodes.operator_node import OperatorJobNode`. ✓ -- `SourceJobNode` referenced in Task 4 — uses `from orcapod.core.nodes.source_node import SourceJobNode`. ✓ -- `bp_node._input_stream` (Task 4, FunctionNodeBase branch) — `FunctionNodeBase` has `_input_stream` attribute. ✓ -- `bp_node._input_streams` (Task 4, OperatorNodeBase branch) — `OperatorNodeBase` has `_input_streams` attribute. ✓ -- `rec_node._function_pod`, `rec_node._table_scope`, `rec_node.tracker_manager` (Task 4) — all present on `FunctionJobNode` via `FunctionNodeBase`. ✓ -- `rec_node._operator`, `rec_node._input_streams`, `rec_node._table_scope`, `rec_node.tracker_manager` (Task 4) — all present on `OperatorJobNode` via `OperatorNodeBase`. ✓ diff --git a/superpowers/plans/2026-05-23-eng-515-pipeline-pipelinejob-refactor.md b/superpowers/plans/2026-05-23-eng-515-pipeline-pipelinejob-refactor.md deleted file mode 100644 index beadf0e83..000000000 --- a/superpowers/plans/2026-05-23-eng-515-pipeline-pipelinejob-refactor.md +++ /dev/null @@ -1,1963 +0,0 @@ -# Pipeline & PipelineJob Invocation-Capture Refactor — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Unify invocation capture into a single `PodInvocation` primitive, move shared recording logic into `AbstractPipelineBase`, and replace the two divergent `compile()` implementations with one class-property–driven pass. - -**Architecture:** Introduce `FunctionInvocation` / `OperatorInvocation` (subclasses of `ContentIdentifiableBase`) as the minimal recording primitive. `AbstractPipelineBase` gains `_invocation_lut` + `_source_streams` (both additive), a concrete `_record_invocation()`, concrete `record_function/operator_pod_invocation()`, abstract class-property node factories, a concrete `compile()`, `InvocationGraph` value object, and `from_invocations()` / `to_invocations()`. `Pipeline` and `PipelineJob` shed their redundant overrides and declare which node classes to use via class attributes. `SourceJobNode.from_stream()` gains three-way input logic. The legacy `_node_lut`, `_upstreams`, `_graph_edges` fields are kept as compiled artifacts (repopulated at end of `compile()`) so that `_build_execution_graph()` needs only a one-line update. - -**Tech Stack:** Python, PyArrow, NetworkX; always run `uv run pytest` (never `pytest` directly). - ---- - -## File Layout - -**New files:** -- `src/orcapod/pipeline/pod_invocation.py` — `PodInvocation`, `FunctionInvocation`, `OperatorInvocation` -- `tests/test_pipeline/test_pod_invocation.py` — unit tests for the new primitives -- `tests/test_pipeline/test_pipeline_base_recording.py` — tests for unified recording path -- `tests/test_pipeline/test_invocation_transitions.py` — tests for to/from_invocations and cross-class transitions - -**Modified files (in execution order):** -1. `src/orcapod/pipeline/base.py` — add `_invocation_lut`, `_source_streams`; concrete recording methods; abstract class-property factories; `InvocationGraph`; `from_invocations()`, `to_invocations()`; concrete `compile()`; `reset()` becomes no-op -2. `src/orcapod/pipeline/graph.py` — remove recording/compile overrides; add class-property attributes -3. `src/orcapod/core/nodes/source_node.py` — add `SourceJobNode.from_stream()` three-way logic -4. `src/orcapod/pipeline/job.py` — remove `_rec_*` fields/methods; add class-property attributes; thin `from_pipeline()`/`as_pipeline()`; lazy `compiled_pipeline`; thin `compile()` override; fix `_build_execution_graph()` one-liner - ---- - -## Critical invariant: invocation hash == node hash - -`StreamBase.identity_structure()` returns `(producer, producer.argument_symmetry(upstreams))` when a producer exists. Both `FunctionNodeBase` and `OperatorNodeBase` use this via `producer` and `upstreams` properties. The `PodInvocation.identity_structure()` returns `(pod, pod.argument_symmetry(input_streams))` — identical structure. So `FunctionInvocation(pod, (stream,)).content_hash() == FunctionNode(function_pod=pod, input_stream=stream).content_hash()`. This invariant means `_invocation_lut` keys match `_persistent_node_map` keys and `_hash_graph` vertex labels. - ---- - -## Task 1: Write all failing tests - -**Files:** -- Create: `tests/test_pipeline/test_pod_invocation.py` -- Create: `tests/test_pipeline/test_pipeline_base_recording.py` -- Create: `tests/test_pipeline/test_invocation_transitions.py` - -- [ ] **Step 1.1: Create test_pod_invocation.py** - -```python -"""Tests for PodInvocation, FunctionInvocation, OperatorInvocation.""" -from __future__ import annotations - -import pytest -import pyarrow as pa - -from orcapod.core.function_pod import FunctionPod -from orcapod.core.operators import Join -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.sources import ArrowTableSource - - -def _make_source(tag_col: str, data_col: str) -> ArrowTableSource: - table = pa.table({ - tag_col: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array([1, 2], type=pa.int64()), - }) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) - - -def _make_function_pod() -> FunctionPod: - def double(value: int) -> int: - return value * 2 - return FunctionPod(PythonDataFunction(double)) - - -class TestFunctionInvocation: - def test_import(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation - assert FunctionInvocation is not None - - def test_hash_stability(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation - pod = _make_function_pod() - stream = _make_source("key", "value") - inv1 = FunctionInvocation(pod=pod, input_streams=(stream,)) - inv2 = FunctionInvocation(pod=pod, input_streams=(stream,)) - assert inv1.content_hash() == inv2.content_hash() - - def test_hash_matches_function_node(self): - """FunctionInvocation hash must equal FunctionNode hash — critical invariant.""" - from orcapod.pipeline.pod_invocation import FunctionInvocation - from orcapod.core.nodes.function_node import FunctionNode - pod = _make_function_pod() - stream = _make_source("key", "value") - inv = FunctionInvocation(pod=pod, input_streams=(stream,)) - node = FunctionNode(function_pod=pod, input_stream=stream) - assert inv.content_hash() == node.content_hash() - - def test_label_stored(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation - pod = _make_function_pod() - stream = _make_source("key", "value") - inv = FunctionInvocation(pod=pod, input_streams=(stream,), label="my_node") - assert inv.label == "my_node" - - def test_pod_and_input_streams_accessible(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation - pod = _make_function_pod() - stream = _make_source("key", "value") - inv = FunctionInvocation(pod=pod, input_streams=(stream,)) - assert inv.pod is pod - assert inv.input_streams == (stream,) - - def test_isinstance_distinguishable_from_operator(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation, OperatorInvocation - pod = _make_function_pod() - stream = _make_source("key", "value") - inv = FunctionInvocation(pod=pod, input_streams=(stream,)) - assert isinstance(inv, FunctionInvocation) - assert not isinstance(inv, OperatorInvocation) - - -class TestOperatorInvocation: - def test_import(self): - from orcapod.pipeline.pod_invocation import OperatorInvocation - assert OperatorInvocation is not None - - def test_hash_stability(self): - from orcapod.pipeline.pod_invocation import OperatorInvocation - join = Join() - s1 = _make_source("key", "value") - s2 = _make_source("key", "score") - inv1 = OperatorInvocation(pod=join, input_streams=(s1, s2)) - inv2 = OperatorInvocation(pod=join, input_streams=(s1, s2)) - assert inv1.content_hash() == inv2.content_hash() - - def test_hash_matches_operator_node(self): - """OperatorInvocation hash must equal OperatorNode hash — critical invariant.""" - from orcapod.pipeline.pod_invocation import OperatorInvocation - from orcapod.core.nodes.operator_node import OperatorNode - join = Join() - s1 = _make_source("key", "value") - s2 = _make_source("key", "score") - inv = OperatorInvocation(pod=join, input_streams=(s1, s2)) - node = OperatorNode(operator=join, input_streams=(s1, s2)) - assert inv.content_hash() == node.content_hash() - - def test_commutative_operator_same_hash_regardless_of_input_order(self): - """Join is commutative — hash must be order-independent.""" - from orcapod.pipeline.pod_invocation import OperatorInvocation - join = Join() - s1 = _make_source("key", "value") - s2 = _make_source("key", "score") - inv_ab = OperatorInvocation(pod=join, input_streams=(s1, s2)) - inv_ba = OperatorInvocation(pod=join, input_streams=(s2, s1)) - assert inv_ab.content_hash() == inv_ba.content_hash() - - def test_isinstance_distinguishable_from_function(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation, OperatorInvocation - join = Join() - s1 = _make_source("key", "value") - s2 = _make_source("key", "score") - inv = OperatorInvocation(pod=join, input_streams=(s1, s2)) - assert isinstance(inv, OperatorInvocation) - assert not isinstance(inv, FunctionInvocation) -``` - -- [ ] **Step 1.2: Create test_pipeline_base_recording.py** - -```python -"""Tests for the unified recording path in AbstractPipelineBase. - -These tests verify the new _record_invocation / _invocation_lut / _source_streams -mechanics by exercising them through Pipeline (concrete subclass). -""" -from __future__ import annotations - -import pytest -import pyarrow as pa - -from orcapod.core.function_pod import FunctionPod -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.operators import Join -from orcapod.core.sources import ArrowTableSource -from orcapod.pipeline import Pipeline - - -def _src(tag_col: str, data_col: str) -> ArrowTableSource: - table = pa.table({ - tag_col: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array([1, 2], type=pa.int64()), - }) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) - - -def _fn(name: str = "double") -> FunctionPod: - def fn(value: int) -> int: - return value * 2 - fn.__name__ = name - return FunctionPod(PythonDataFunction(fn)) - - -class TestRecordFunctionPodInvocation: - def test_record_function_pod_adds_to_invocation_lut(self): - """After recording a function pod, _invocation_lut must contain the invocation.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test", auto_compile=False) - with pipeline: - pod(stream, label="out") - # _invocation_lut is the new field; _node_lut is the legacy compiled field - assert len(pipeline._invocation_lut) == 1 - - def test_record_function_pod_captures_source_stream(self): - """The upstream concrete stream must land in _source_streams.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test", auto_compile=False) - with pipeline: - pod(stream, label="out") - stream_hash = stream.content_hash().to_string() - assert stream_hash in pipeline._source_streams - - def test_invocation_lut_additive_across_with_blocks(self): - """Opening a second with-block appends to _invocation_lut, not replaces.""" - pod1 = _fn("fn1") - pod2 = _fn("fn2") - s = _src("key", "value") - pipeline = Pipeline(name="test", auto_compile=False) - with pipeline: - out1 = pod1(s, label="step1") - with pipeline: - pod2(out1, label="step2") - assert len(pipeline._invocation_lut) == 2 - - def test_compile_rebuilds_persistent_node_map_from_scratch(self): - """compile() must produce correct _persistent_node_map from _invocation_lut.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - assert "out" in pipeline._nodes - assert len(pipeline._persistent_node_map) == 2 # 1 source + 1 function node - - def test_compile_creates_source_node_for_unregistered_upstream(self): - """Streams with no recorded invocation must become SourceNode at compile time.""" - from orcapod.core.nodes.source_node import SourceNode - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - source_hash = stream.content_hash().to_string() - assert isinstance(pipeline._persistent_node_map[source_hash], SourceNode) - - -class TestRecordOperatorPodInvocation: - def test_record_operator_pod_adds_to_invocation_lut(self): - """After recording an operator, _invocation_lut must contain the invocation.""" - join = Join() - s1 = _src("key", "value") - s2 = _src("key", "score") - pipeline = Pipeline(name="test", auto_compile=False) - with pipeline: - join(s1, s2, label="joined") - assert len(pipeline._invocation_lut) == 1 - - def test_record_operator_captures_both_source_streams(self): - join = Join() - s1 = _src("key", "value") - s2 = _src("key", "score") - pipeline = Pipeline(name="test", auto_compile=False) - with pipeline: - join(s1, s2, label="joined") - h1 = s1.content_hash().to_string() - h2 = s2.content_hash().to_string() - assert h1 in pipeline._source_streams - assert h2 in pipeline._source_streams -``` - -- [ ] **Step 1.3: Create test_invocation_transitions.py** - -```python -"""Tests for to_invocations / from_invocations and cross-class transitions.""" -from __future__ import annotations - -import pytest -import pyarrow as pa - -from orcapod.core.function_pod import FunctionPod -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.operators import Join -from orcapod.core.sources import ArrowTableSource -from orcapod.core.nodes.source_node import SourceNode, SourceJobNode -from orcapod.pipeline import Pipeline -from orcapod.pipeline.job import PipelineJob - - -def _src(tag_col: str, data_col: str) -> ArrowTableSource: - table = pa.table({ - tag_col: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array([1, 2], type=pa.int64()), - }) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) - - -def _fn() -> FunctionPod: - def double(value: int) -> int: - return value * 2 - return FunctionPod(PythonDataFunction(double)) - - -class TestToInvocations: - def test_returns_invocation_graph(self): - from orcapod.pipeline.base import InvocationGraph - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - graph = pipeline.to_invocations() - assert isinstance(graph, InvocationGraph) - - def test_invocation_graph_contains_one_invocation(self): - from orcapod.pipeline.pod_invocation import FunctionInvocation - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - graph = pipeline.to_invocations() - assert len(graph.invocations) == 1 - assert isinstance(graph.invocations[0], FunctionInvocation) - - def test_invocation_graph_source_streams_has_source_node(self): - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - graph = pipeline.to_invocations() - assert len(graph.source_streams) == 1 - source = list(graph.source_streams.values())[0] - assert isinstance(source, SourceNode) - - -class TestFromInvocations: - def test_pipeline_from_invocations_roundtrip(self): - """Pipeline → to_invocations() → Pipeline.from_invocations() must produce equivalent graph.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - graph = pipeline.to_invocations() - pipeline2 = Pipeline.from_invocations(graph) - assert "out" in pipeline2._nodes - assert len(pipeline2._persistent_node_map) == len(pipeline._persistent_node_map) - - def test_pipeline_to_job_via_from_invocations(self): - """PipelineJob.from_invocations(pipeline.to_invocations()) must produce bound-ready job.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - job = PipelineJob.from_invocations(pipeline.to_invocations()) - assert "out" in job._nodes - # Source nodes should be unbound SourceJobNodes - source_nodes = [ - n for n in job._persistent_node_map.values() - if isinstance(n, SourceJobNode) - ] - assert len(source_nodes) == 1 - assert source_nodes[0].bound_source is None - - -class TestFromPipeline: - def test_from_pipeline_thin_composition(self): - """PipelineJob.from_pipeline(pipeline) must produce the same result as going via invocations.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - job = PipelineJob.from_pipeline(pipeline) - assert "out" in job._nodes - - def test_as_pipeline_thin_composition(self): - """job.as_pipeline() must produce a Pipeline structurally equivalent to the original.""" - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - job = PipelineJob.from_pipeline(pipeline) - pipeline2 = job.as_pipeline() - assert "out" in pipeline2._nodes - - -class TestSourceJobNodeFromStream: - def test_sjn_from_sjn_copies_bound_source(self): - """SJN → SJN: bound_source is copied; the input SJN is NOT used as bound_source.""" - concrete = _src("key", "value") - sjn = SourceJobNode( - name="data", - tag_schema=concrete.output_schema()[0], - data_schema=concrete.output_schema()[1], - bound_source=concrete, - ) - sjn2 = SourceJobNode.from_stream(sjn) - assert isinstance(sjn2, SourceJobNode) - assert sjn2.bound_source is concrete # bound_source copied, not sjn itself - assert sjn2.bound_source is not sjn # sjn itself is NOT the bound_source - - def test_sjn_from_source_node_creates_unbound(self): - """SourceNode → SJN: creates unbound SourceJobNode with matching schema.""" - from orcapod.types import Schema - sn = SourceNode( - name="data", - tag_schema=Schema({"key": str}), - data_schema=Schema({"value": int}), - ) - sjn = SourceJobNode.from_stream(sn) - assert isinstance(sjn, SourceJobNode) - assert sjn.bound_source is None - assert sjn.name == "data" - - def test_sjn_from_concrete_stream_creates_bound(self): - """Concrete stream → SJN: creates bound SourceJobNode; hash equals bound source hash.""" - concrete = _src("key", "value") - sjn = SourceJobNode.from_stream(concrete) - assert isinstance(sjn, SourceJobNode) - assert sjn.bound_source is concrete - assert sjn.content_hash() == concrete.content_hash() - - def test_unbound_sjn_hash_equals_source_node_hash(self): - """Unbound SJN must have same content_hash as the corresponding SourceNode.""" - from orcapod.types import Schema - sn = SourceNode( - name="data", - tag_schema=Schema({"key": str}), - data_schema=Schema({"value": int}), - ) - sjn = SourceJobNode.from_stream(sn) - assert sjn.content_hash() == sn.content_hash() - - -class TestCrossWithBlockReconnection: - def test_pipelinejob_same_concrete_source_in_two_blocks_same_hash(self): - """PipelineJob: same concrete source in two with-blocks produces the same source node hash.""" - pod1 = _fn() - pod2 = _fn() - - def double2(value: int) -> int: - return value * 3 - - pod2 = FunctionPod(PythonDataFunction(double2)) - concrete = _src("key", "value") - - job = PipelineJob(name="test", auto_compile=False) - with job: - out1 = pod1(concrete, label="step1") - hash_after_first = concrete.content_hash().to_string() - source_hash_first = list( - h for h, n in job._persistent_node_map.items() - if isinstance(n, SourceJobNode) - )[0] - with job: - pod2(concrete, label="step2") - source_hash_second = list( - h for h, n in job._persistent_node_map.items() - if isinstance(n, SourceJobNode) - )[0] - assert source_hash_first == source_hash_second - - def test_pipeline_source_hash_after_compile_differs_from_concrete_stream_hash(self): - """Pipeline: SourceNode hash is schema-based, NOT the original concrete stream hash.""" - from orcapod.core.nodes.source_node import SourceNode - pod = _fn() - stream = _src("key", "value") - pipeline = Pipeline(name="test") - with pipeline: - pod(stream, label="out") - original_hash = stream.content_hash().to_string() - # The source node in persistent_node_map should be a SourceNode whose hash - # differs from the original concrete stream hash (schema-based normalization) - source_nodes = { - h: n for h, n in pipeline._persistent_node_map.items() - if isinstance(n, SourceNode) - } - assert len(source_nodes) == 1 - source_node_hash = list(source_nodes.keys())[0] - assert source_node_hash != original_hash -``` - -- [ ] **Step 1.4: Run all three test files and confirm they fail** - -```bash -cd /home/kurouto/kurouto-jobs/c6a75f31-838c-4e23-9ec2-377eecdee49e/orcapod-python -uv run pytest tests/test_pipeline/test_pod_invocation.py tests/test_pipeline/test_pipeline_base_recording.py tests/test_pipeline/test_invocation_transitions.py -v 2>&1 | head -60 -``` - -Expected: collection errors or `ImportError: cannot import name 'FunctionInvocation'`. - ---- - -## Task 2: Create `pod_invocation.py` - -**Files:** -- Create: `src/orcapod/pipeline/pod_invocation.py` - -- [ ] **Step 2.1: Write pod_invocation.py** - -```python -"""PodInvocation — minimal recording primitive for Pipeline and PipelineJob. - -Records a single pod invocation (function or operator) against one or more -input streams. Provides content-addressable identity so that the same -logical invocation always hashes to the same value regardless of the -recording order. - -``identity_structure()`` mirrors ``StreamBase.identity_structure()`` — -``(pod, pod.argument_symmetry(input_streams))`` — which ensures that -``FunctionInvocation.content_hash() == FunctionNode.content_hash()`` and -``OperatorInvocation.content_hash() == OperatorNode.content_hash()`` for -the same pod and input streams. This invariant lets ``_invocation_lut`` -keys be reused as ``_persistent_node_map`` keys without a separate -translation step. -""" -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from orcapod.core.base import ContentIdentifiableBase - -if TYPE_CHECKING: - from orcapod.protocols.core_protocols import ( - FunctionPodProtocol, - OperatorPodProtocol, - StreamProtocol, - ) - - -class PodInvocation(ContentIdentifiableBase): - """Abstract recording primitive for a pod applied to input streams. - - Args: - pod: The pod being invoked (function or operator). - input_streams: Tuple of upstream streams passed to the pod. - label: Optional display label for the resulting compiled node. - """ - - def __init__( - self, - pod: Any, - input_streams: "tuple[StreamProtocol, ...]", - label: str | None = None, - ) -> None: - super().__init__() - self._pod = pod - self._input_streams = tuple(input_streams) - self._label = label - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def pod(self) -> Any: - """The pod being invoked.""" - return self._pod - - @property - def input_streams(self) -> "tuple[StreamProtocol, ...]": - """Upstream streams passed to this invocation.""" - return self._input_streams - - @property - def label(self) -> str | None: - """Optional display label for the compiled node.""" - return self._label - - # ------------------------------------------------------------------ - # Identity — mirrors StreamBase.identity_structure() - # ------------------------------------------------------------------ - - def identity_structure(self) -> Any: - """Return ``(pod, pod.argument_symmetry(input_streams))``. - - This matches the identity structure of the corresponding compiled - node, ensuring hash equality between invocation and node. - """ - return (self._pod, self._pod.argument_symmetry(self._input_streams)) - - def pipeline_identity_structure(self) -> Any: - """Return same as ``identity_structure()``.""" - return self.identity_structure() - - -class FunctionInvocation(PodInvocation): - """Invocation of a function pod against a single input stream. - - Args: - pod: A ``FunctionPodProtocol`` instance. - input_streams: Tuple with exactly one stream. - label: Optional display label. - """ - - def __init__( - self, - pod: "FunctionPodProtocol", - input_streams: "tuple[StreamProtocol, ...]", - label: str | None = None, - ) -> None: - if len(input_streams) != 1: - raise ValueError( - f"FunctionInvocation requires exactly 1 input stream; " - f"got {len(input_streams)}." - ) - super().__init__(pod=pod, input_streams=input_streams, label=label) - - -class OperatorInvocation(PodInvocation): - """Invocation of an operator pod against one or more input streams. - - Args: - pod: An ``OperatorPodProtocol`` instance. - input_streams: Tuple with one or more streams. - label: Optional display label. - """ - - def __init__( - self, - pod: "OperatorPodProtocol", - input_streams: "tuple[StreamProtocol, ...]", - label: str | None = None, - ) -> None: - if len(input_streams) == 0: - raise ValueError("OperatorInvocation requires at least 1 input stream.") - super().__init__(pod=pod, input_streams=input_streams, label=label) -``` - -- [ ] **Step 2.2: Run PodInvocation tests — confirm tests 8a pass** - -```bash -uv run pytest tests/test_pipeline/test_pod_invocation.py -v -``` - -Expected: all pass. - -- [ ] **Step 2.3: Commit** - -```bash -git add src/orcapod/pipeline/pod_invocation.py tests/test_pipeline/test_pod_invocation.py -git commit -m "feat(pipeline): add PodInvocation, FunctionInvocation, OperatorInvocation primitives" -``` - ---- - -## Task 3: Update `AbstractPipelineBase` - -**Files:** -- Modify: `src/orcapod/pipeline/base.py` - -This is the central change. Read `src/orcapod/pipeline/base.py` before editing. - -The design uses a **separate** `_invocation_lut` field (stores `PodInvocation` objects during recording) rather than overloading `_node_lut`. After `compile()`, the legacy fields `_node_lut`, `_upstreams`, and `_graph_edges` are repopulated from `_persistent_node_map` and `_hash_graph` so that `_build_execution_graph()` in `PipelineJob` continues to work without modification. - -- [ ] **Step 3.1: Replace base.py with the updated version** - -Replace the entire file `src/orcapod/pipeline/base.py` with: - -```python -"""AbstractPipelineBase — shared recording mechanism for Pipeline and PipelineJob.""" -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -from orcapod.core.tracker import AutoRegisteringContextBasedTracker -from orcapod.pipeline.pod_invocation import ( - FunctionInvocation, - OperatorInvocation, - PodInvocation, -) -from orcapod.protocols import core_protocols as cp -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import networkx as nx -else: - nx = LazyModule("networkx") - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class InvocationGraph: - """Interchange value object between Pipeline and PipelineJob. - - Carries a topologically ordered tuple of ``PodInvocation`` objects and a - mapping from content-hash string to the corresponding source stream. - Both classes exchange representations by converting to this neutral form - via ``to_invocations()`` and reconstructing via ``from_invocations()``. - - Args: - invocations: Topologically ordered sequence of pod invocations - (sources excluded). - source_streams: Mapping of content-hash string → source stream - (nodes whose invocation was not recorded in the pipeline). - """ - - invocations: tuple[PodInvocation, ...] - source_streams: dict[str, Any] # hash → StreamProtocol-compatible node - - -class AbstractPipelineBase(AutoRegisteringContextBasedTracker, ABC): - """Shared recording mechanism and graph state for Pipeline and PipelineJob. - - Manages the ``with``-block recording phase: accumulating invocations into - ``_invocation_lut``, capturing raw source streams into ``_source_streams``, - and building the topology in ``_hash_graph``. On context exit, ``compile()`` - materialises the accumulated state into a frozen DAG of node objects. - - ``_invocation_lut`` and ``_source_streams`` are **additive** — they persist - across multiple ``with`` blocks so that repeated recording sessions extend - the same graph rather than replacing it. - - Args: - name: Pipeline name (string or tuple). Used to scope database paths. - tracker_manager: Optional tracker manager override. - """ - - def __init__( - self, - name: str | tuple[str, ...] = "pipeline", - tracker_manager: cp.TrackerManagerProtocol | None = None, - ) -> None: - super().__init__(tracker_manager=tracker_manager) - self._name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) - - # --- Additive recording state (never cleared) ----------------- - # Maps content-hash-string → PodInvocation for each recorded invocation. - self._invocation_lut: dict[str, PodInvocation] = {} - # Maps content-hash-string → raw stream for each unregistered upstream. - self._source_streams: dict[str, Any] = {} - # Topology graph — vertices and edges are content-hash strings. - # Additive: persists and grows across multiple with-blocks. - self._hash_graph: "nx.DiGraph" = nx.DiGraph() - - # --- Compiled state (populated / replaced by compile()) -------- - self._persistent_node_map: dict[str, Any] = {} - self._nodes: dict[str, Any] = {} - self._node_graph: "nx.DiGraph | None" = None - self._compiled: bool = False - - # --- Legacy fields kept for _build_execution_graph() compat ---- - # Populated from _persistent_node_map / _hash_graph at end of compile(). - self._node_lut: dict[str, Any] = {} # hash → non-source compiled node - self._upstreams: dict[str, Any] = {} # hash → source compiled node - self._graph_edges: list[tuple[str, str]] = [] # edge list from _hash_graph - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def name(self) -> tuple[str, ...]: - """Pipeline name tuple.""" - return self._name - - @property - def graph(self) -> "nx.DiGraph": - """Directed hash graph of accumulated pipeline structure.""" - return self._hash_graph - - @property - def compiled_nodes(self) -> dict[str, Any]: - """Copy of the compiled nodes dict (label → node).""" - return self._nodes.copy() - - # ------------------------------------------------------------------ - # Abstract — subclass node-factory declarations - # ------------------------------------------------------------------ - - @property - @abstractmethod - def source_node_class(self) -> type: - """Node class to use for source (leaf) nodes — e.g. ``SourceNode``.""" - ... - - @property - @abstractmethod - def function_node_class(self) -> type: - """Node class to use for function-pod invocations — e.g. ``FunctionNode``.""" - ... - - @property - @abstractmethod - def operator_node_class(self) -> type: - """Node class to use for operator-pod invocations — e.g. ``OperatorNode``.""" - ... - - # ------------------------------------------------------------------ - # Recording - # ------------------------------------------------------------------ - - def record_function_pod_invocation( - self, - pod: "cp.FunctionPodProtocol", - input_stream: "cp.StreamProtocol", - label: str | None = None, - ) -> None: - """Record a function pod invocation into the graph. - - Args: - pod: The function pod being invoked. - input_stream: The upstream stream. - label: Optional display label for the resulting compiled node. - """ - self._record_invocation(FunctionInvocation(pod=pod, input_streams=(input_stream,), label=label)) - - def record_operator_pod_invocation( - self, - pod: "cp.OperatorPodProtocol", - upstreams: "tuple[cp.StreamProtocol, ...]" = (), - label: str | None = None, - ) -> None: - """Record an operator pod invocation into the graph. - - Args: - pod: The operator pod being invoked. - upstreams: Upstream streams for this operator. - label: Optional display label for the resulting compiled node. - """ - self._record_invocation(OperatorInvocation(pod=pod, input_streams=tuple(upstreams), label=label)) - - def _record_invocation(self, invocation: PodInvocation) -> None: - """Store *invocation* and update the topology graph. - - The content hash of a ``PodInvocation`` equals the content hash of the - corresponding compiled node (``FunctionNode`` / ``OperatorNode``), so the - same hash key works in both ``_invocation_lut`` and ``_persistent_node_map``. - - Args: - invocation: The pod invocation to record. - """ - key = invocation.content_hash().to_string() - self._invocation_lut[key] = invocation - self._hash_graph.add_node(key) - for upstream in invocation.input_streams: - upstream_hash = upstream.content_hash().to_string() - self._hash_graph.add_edge(upstream_hash, key) - if upstream_hash not in self._source_streams: - self._source_streams[upstream_hash] = upstream - - # ------------------------------------------------------------------ - # reset() — no-op after refactor - # ------------------------------------------------------------------ - - def reset(self) -> None: - """No-op — all recording state is additive and persists across with-blocks. - - Note: - ``_invocation_lut``, ``_source_streams``, and ``_hash_graph`` intentionally - accumulate across ``with`` blocks. ``_persistent_node_map``, ``_node_lut``, - ``_upstreams``, and ``_graph_edges`` are compiled artifacts overwritten by - each ``compile()`` call. Nothing needs to be cleared on re-entry. - """ - - def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: - """Exit the recording context, compiling if no exception occurred.""" - super().__exit__(exc_type, exc_value, traceback) - if exc_type is None: - self.compile() - - def __getattr__(self, item: str) -> Any: - """Look up compiled nodes by label as attribute access.""" - if item.startswith("_"): - raise AttributeError(item) - nodes = object.__getattribute__(self, "_nodes") - if item in nodes: - return nodes[item] - raise AttributeError( - f"{type(self).__name__!r} has no attribute {item!r}. " - f"Available node labels: {sorted(nodes.keys())}" - ) - - # ------------------------------------------------------------------ - # compile() — single-pass, class-property driven - # ------------------------------------------------------------------ - - def compile(self) -> None: - """Compile recorded invocations into a frozen DAG. - - Walks ``_hash_graph`` topologically. Streams that appear as inputs - but have no registered invocation are promoted to source nodes via - ``self.source_node_class.from_stream()``. Each ``PodInvocation`` in - ``_invocation_lut`` becomes a ``function_node_class`` or - ``operator_node_class`` instance. - - After compilation: - - ``_persistent_node_map`` — hash → compiled node (all node types) - - ``_nodes`` — label → compiled node (labelled nodes only) - - ``_node_graph`` — nx.DiGraph with node objects as vertices - - Legacy fields ``_node_lut``, ``_upstreams``, ``_graph_edges`` - are repopulated for backward compat with ``_build_execution_graph()``. - - ``_compiled`` is set to ``True``. - - This method always rebuilds from scratch; it does NOT perform - incremental compilation. - """ - import networkx as _nx - - source_node_cls = self.source_node_class - - # 1. Source hashes: inputs that have no registered invocation. - source_hashes = set(self._source_streams.keys()) - set(self._invocation_lut.keys()) - - # 2. Create source nodes. - node_map: dict[str, Any] = { - h: source_node_cls.from_stream(self._source_streams[h]) - for h in source_hashes - if h in self._source_streams - } - - # 3. Topological pass — create function / operator nodes. - for key in _nx.topological_sort(self._hash_graph): - if key in node_map: - continue # already a source node - if key not in self._invocation_lut: - continue # vertex in hash_graph with no invocation (e.g. pure source hash) - inv = self._invocation_lut[key] - upstream_nodes = [ - node_map[up.content_hash().to_string()] - for up in inv.input_streams - ] - if isinstance(inv, FunctionInvocation): - node_map[key] = self.function_node_class( - function_pod=inv.pod, - input_stream=upstream_nodes[0], - label=inv.label, - ) - else: - node_map[key] = self.operator_node_class( - operator=inv.pod, - input_streams=tuple(upstream_nodes), - label=inv.label, - ) - - self._persistent_node_map = node_map - - # 4. Label disambiguation (preserves existing Pipeline.compile() behavior). - name_candidates: dict[str, list] = {} - for node in node_map.values(): - name_candidates.setdefault(node.label, []).append(node) - - self._nodes.clear() - for label, nodes in name_candidates.items(): - if len(nodes) > 1: - sorted_nodes = sorted(nodes, key=lambda n: n.content_hash().to_string()) - for i, node in enumerate(sorted_nodes, start=1): - key = f"{label}_{i}" - self._nodes[key] = node - node._label = key - else: - self._nodes[label] = nodes[0] - - # 5. Build node_graph (DiGraph with node objects as vertices). - self._node_graph = _nx.DiGraph() - for up_hash, down_hash in self._hash_graph.edges(): - up_node = node_map.get(up_hash) - down_node = node_map.get(down_hash) - if up_node is not None and down_node is not None: - self._node_graph.add_edge(up_node, down_node) - for node in node_map.values(): - if node not in self._node_graph: - self._node_graph.add_node(node) - - # 6. Enrich hash_graph node attributes (used by GraphRenderer and serialization). - for node_hash, node in node_map.items(): - if node_hash not in self._hash_graph: - continue - attrs = self._hash_graph.nodes[node_hash] - if not attrs.get("node_type"): - attrs["node_type"] = node.node_type - if not attrs.get("label"): - computed = node._label or ( - node.computed_label() if hasattr(node, "computed_label") else None - ) - if computed: - attrs["label"] = computed - if not attrs.get("pipeline_hash"): - attrs["pipeline_hash"] = node.pipeline_hash().to_string() - - # 7. Populate legacy fields for _build_execution_graph() backward compat. - self._node_lut = { - h: n for h, n in node_map.items() - if not isinstance(n, source_node_cls) - } - self._upstreams = { - h: n for h, n in node_map.items() - if isinstance(n, source_node_cls) - } - self._graph_edges = list(self._hash_graph.edges()) - - self._compiled = True - - # ------------------------------------------------------------------ - # InvocationGraph transitions - # ------------------------------------------------------------------ - - def to_invocations(self) -> InvocationGraph: - """Extract an ``InvocationGraph`` from the compiled ``_persistent_node_map``. - - Reconstructs from ``_persistent_node_map`` (not raw ``_invocation_lut``) so - that the in-memory path and the save/load path produce consistent results. - Source nodes (leaves) go into ``source_streams``; function and operator - nodes become ``FunctionInvocation`` / ``OperatorInvocation`` objects. - - Returns: - An ``InvocationGraph`` with topologically ordered invocations and - a mapping of hash → source node (which acts as its own stream). - - Raises: - RuntimeError: If the pipeline has not been compiled. - """ - import networkx as _nx - - if not self._compiled: - raise RuntimeError( - "Cannot call to_invocations() before compile(). " - "Use 'with pipeline:' or call compile() first." - ) - - source_node_cls = self.source_node_class - fn_node_cls = self.function_node_class - source_streams: dict[str, Any] = {} - invocations: list[PodInvocation] = [] - - for node_hash, node in self._persistent_node_map.items(): - if isinstance(node, source_node_cls): - source_streams[node_hash] = node # source node IS its own stream - elif isinstance(node, fn_node_cls): - invocations.append( - FunctionInvocation( - pod=node._function_pod, - input_streams=(node.upstreams[0],), - label=node._label, - ) - ) - else: - invocations.append( - OperatorInvocation( - pod=node._operator, - input_streams=tuple(node.upstreams), - label=node._label, - ) - ) - - # Sort topologically using _hash_graph. - topo_keys = list(_nx.topological_sort(self._hash_graph)) - inv_by_hash = {inv.content_hash().to_string(): inv for inv in invocations} - ordered = [inv_by_hash[k] for k in topo_keys if k in inv_by_hash] - - return InvocationGraph( - invocations=tuple(ordered), - source_streams=source_streams, - ) - - @classmethod - def from_invocations( - cls, - graph: InvocationGraph, - name: str | tuple[str, ...] = "pipeline", - ) -> "AbstractPipelineBase": - """Reconstruct a Pipeline or PipelineJob from an ``InvocationGraph``. - - Calls ``cls(name=name)`` to properly initialise all fields, then - populates ``_invocation_lut``, ``_source_streams``, and ``_hash_graph`` - from *graph* before calling ``compile()``. - - This classmethod lives on ``AbstractPipelineBase`` (not on the - subclasses) so that neither ``Pipeline`` nor ``PipelineJob`` needs - to access the other's private state directly. - - Args: - graph: The ``InvocationGraph`` to reconstruct from. - name: Pipeline name for the new instance. - - Returns: - A compiled instance of ``cls``. - """ - instance = cls(name=name) - instance._invocation_lut = { - inv.content_hash().to_string(): inv - for inv in graph.invocations - } - instance._source_streams = dict(graph.source_streams) - for inv in graph.invocations: - instance._hash_graph.add_node(inv.content_hash().to_string()) - for upstream in inv.input_streams: - instance._hash_graph.add_edge( - upstream.content_hash().to_string(), - inv.content_hash().to_string(), - ) - # Also add source hashes as isolated nodes (no incoming edges) - for h in graph.source_streams: - if h not in instance._hash_graph: - instance._hash_graph.add_node(h) - instance.compile() - return instance -``` - -- [ ] **Step 3.2: Run base recording tests** - -```bash -uv run pytest tests/test_pipeline/test_pipeline_base_recording.py -v -``` - -Expected: most or all pass (Pipeline still has its own compile() — that gets removed in Task 4, but the recording path is now live). - -Note: `test_invocation_lut_additive_across_with_blocks` will only pass once Pipeline's `__enter__` no longer calls `reset()` (it doesn't — Pipeline has no `__enter__` override, and `AutoRegisteringContextBasedTracker.__enter__` does not call `reset()`). If tests fail due to compile override still present in Pipeline, they will be fixed in Task 4. - -- [ ] **Step 3.3: Commit base changes** - -```bash -git add src/orcapod/pipeline/base.py tests/test_pipeline/test_pipeline_base_recording.py tests/test_pipeline/test_invocation_transitions.py -git commit -m "refactor(pipeline): unify recording in AbstractPipelineBase with _invocation_lut and shared compile()" -``` - ---- - -## Task 4: Update `Pipeline` - -**Files:** -- Modify: `src/orcapod/pipeline/graph.py` - -Read `src/orcapod/pipeline/graph.py` before editing. The Pipeline class starts at line 34. The GraphRenderer and rendering utilities begin at line 625 — **leave them entirely unchanged.** - -- [ ] **Step 4.1: Remove recording overrides and compile() from Pipeline; add class-property attributes** - -Replace the entire `Pipeline` class (lines 34–618 in `graph.py`, i.e., the `Pipeline` class definition through `_clone_for_execution` and `__dir__`, but before `GraphRenderer`) with the following. The GraphRenderer section that starts at the comment `# ===========================================================================` stays unchanged. - -```python -class Pipeline(AbstractPipelineBase): - """A pure computational blueprint recording operator and function pod invocations. - - During the ``with`` block, operator and function pod invocations are - recorded into an internal graph via the unified ``_record_invocation()`` - path inherited from ``AbstractPipelineBase``. On context exit, - ``compile()`` (also inherited) rewires the graph into a frozen DAG: - - - Leaf streams not registered as invocations → ``SourceNode`` declarations - - Function pod invocations → ``FunctionNode`` - - Operator invocations → ``OperatorNode`` - - To run a ``Pipeline``, use - ``PipelineJob.from_pipeline(pipeline, sources=..., store=...)`` to create - a ``PipelineJob``. - - Args: - name: Pipeline name (string or tuple). Used as the path prefix for - all cache/pipeline paths when the pipeline is run via a - ``PipelineJob``. - auto_compile: If ``True`` (default), ``compile()`` is called - automatically when the context manager exits. - """ - - # ------------------------------------------------------------------ - # Node-factory class attributes (used by AbstractPipelineBase.compile()) - # ------------------------------------------------------------------ - - @property - def source_node_class(self) -> type: - from orcapod.core.nodes.source_node import SourceNode - return SourceNode - - @property - def function_node_class(self) -> type: - from orcapod.core.nodes.function_node import FunctionNode - return FunctionNode - - @property - def operator_node_class(self) -> type: - from orcapod.core.nodes.operator_node import OperatorNode - return OperatorNode - - def __init__( - self, - name: str | tuple[str, ...], - tracker_manager: cp.TrackerManagerProtocol | None = None, - auto_compile: bool = True, - ) -> None: - """Initialize a pure computational blueprint pipeline. - - Args: - name: Pipeline name (string or tuple). - tracker_manager: Optional tracker manager override. - auto_compile: If ``True`` (default), ``compile()`` is called - automatically when the context manager exits. - """ - super().__init__(name=name, tracker_manager=tracker_manager) - self._auto_compile = auto_compile - - @property - def nodes(self) -> list: - """Return the list of compiled non-source nodes (by label).""" - return list(self._persistent_node_map.values()) - - # ------------------------------------------------------------------ - # Context manager — respects auto_compile flag - # ------------------------------------------------------------------ - - def __exit__(self, exc_type=None, exc_value=None, traceback=None): - # Call AutoRegisteringContextBasedTracker.__exit__ directly so we can - # gate compile() on the auto_compile flag, bypassing the base class - # __exit__ which always calls compile(). - AutoRegisteringContextBasedTracker.__exit__(self, exc_type, exc_value, traceback) - if exc_type is None and self._auto_compile: - self.compile() - - # ------------------------------------------------------------------ - # Graph display - # ------------------------------------------------------------------ - - def show_graph(self, **kwargs) -> str | None: - """Render the pipeline's node graph. - - Args: - **kwargs: Forwarded to ``render_graph``. - - Raises: - RuntimeError: If the pipeline has not been compiled yet. - """ - if self._node_graph is None: - raise RuntimeError("Pipeline must be compiled before showing the graph.") - return render_graph(self._node_graph, **kwargs) - - # ------------------------------------------------------------------ - # Serialization - # ------------------------------------------------------------------ - - def save(self, path: "str | Path") -> None: - """Serialize the pure pipeline blueprint to a JSON file. - - Saves the full pipeline topology: SourceNode declarations, function - and operator pod configurations, and all edge connections. Runtime - state — databases, execution context, and run metadata — is not - persisted. - - Args: - path: File path to write JSON output to. - - Raises: - ValueError: If the pipeline has not been compiled. - """ - if not self._compiled: - raise ValueError( - "Pipeline is not compiled. Call compile() or use " - "auto_compile=True before saving." - ) - - import json as _json - from orcapod.pipeline.serialization import ( - PIPELINE_FORMAT_VERSION, - serialize_schema, - ) - from orcapod.core.nodes import OperatorNode, FunctionNode - from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass - - nodes: dict[str, Any] = {} - for content_hash_str, node in self._persistent_node_map.items(): - tag_schema, data_schema = node.output_schema() - try: - type_converter = node.data_context.type_converter - except (AttributeError, TypeError): - from orcapod.contexts import resolve_context - type_converter = resolve_context(None).type_converter - - try: - data_context_key = node.data_context_key - except (AttributeError, TypeError): - _dc = getattr(node, "_data_context", None) - data_context_key = _dc.context_key if _dc is not None else None - - descriptor: dict[str, Any] = { - "node_type": node.node_type, - "label": node.label, - "content_hash": node.content_hash().to_string(), - "pipeline_hash": node.pipeline_hash().to_string(), - "output_schema": { - "tag": serialize_schema(tag_schema, type_converter), - "data": serialize_schema(data_schema, type_converter), - }, - "node_uri": list(node.node_uri), - "data_context_key": data_context_key, - } - - match node: - case SourceNodeClass(): - descriptor["source_config"] = { - "source_type": "node", - "name": node.name, - "tag_schema": serialize_schema(node.tag_schema, type_converter), - "data_schema": serialize_schema(node.data_schema, type_converter), - } - descriptor["reconstructable"] = True - - case FunctionNode(): - if node._function_pod is not None: - descriptor["function_config"] = node._function_pod.to_config() - descriptor["table_scope"] = node._table_scope - - case OperatorNode(): - if node._operator is not None: - descriptor["operator_config"] = node._operator.to_config() - descriptor["table_scope"] = node._table_scope - - nodes[content_hash_str] = descriptor - - output: dict[str, Any] = { - "orcapod_pipeline_version": PIPELINE_FORMAT_VERSION, - "pipeline": {"name": list(self._name)}, - "nodes": nodes, - "edges": [list(edge) for edge in self._graph_edges], - } - - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - _json.dump(output, f, indent=2) - - @classmethod - def load(cls, path: "str | Path") -> "Pipeline": - """Deserialize a pure pipeline blueprint from a JSON file. - - Reconstructs topology and SourceNode declarations. The loaded - pipeline is topology-only — to run it, use - ``PipelineJob.from_pipeline(pipeline, sources=..., store=...)``. - - Args: - path: Path to the JSON file produced by :meth:`save`. - - Returns: - A compiled ``Pipeline`` instance with SourceNode leaf nodes. - - Raises: - ValueError: If the file's format version is unsupported. - """ - import json as _json - from orcapod.pipeline.serialization import ( - SUPPORTED_FORMAT_VERSIONS, - deserialize_schema, - ) - from orcapod.core.nodes import FunctionNode, OperatorNode - from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass - from orcapod.types import Schema - - path = Path(path) - with open(path) as f: - data = _json.load(f) - - version = data.get("orcapod_pipeline_version", "") - if version not in SUPPORTED_FORMAT_VERSIONS: - raise ValueError( - f"Unsupported pipeline format version {version!r}. " - f"Supported: {sorted(SUPPORTED_FORMAT_VERSIONS)}" - ) - - pipeline_meta = data["pipeline"] - name = tuple(pipeline_meta["name"]) - nodes_data = data["nodes"] - edges = data["edges"] - - edge_graph: "nx.DiGraph" = nx.DiGraph() - for up_hash, down_hash in edges: - edge_graph.add_edge(up_hash, down_hash) - for node_hash in nodes_data: - if node_hash not in edge_graph: - edge_graph.add_node(node_hash) - topo_order = list(nx.topological_sort(edge_graph)) - - upstream_map: dict[str, list[str]] = {} - for up_hash, down_hash in edges: - upstream_map.setdefault(down_hash, []).append(up_hash) - - reconstructed: dict[str, GraphNode] = {} - - for node_hash in topo_order: - descriptor = nodes_data.get(node_hash) - if descriptor is None: - continue - - node_type = descriptor.get("node_type") - source_config = descriptor.get("source_config") or {} - - if node_type == "source": - source_type = source_config.get("source_type") - if source_type == "node": - node_name = source_config.get("name") or source_config.get("node_name") - if not node_name: - node_name = descriptor.get("label") or "unknown" - if "tag_schema" in source_config and "data_schema" in source_config: - tag_schema = Schema(deserialize_schema(source_config["tag_schema"])) - data_schema = Schema(deserialize_schema(source_config["data_schema"])) - else: - tag_schema = Schema(deserialize_schema(descriptor["output_schema"]["tag"])) - data_schema = Schema(deserialize_schema(descriptor["output_schema"]["data"])) - node = SourceNodeClass( - name=node_name, - tag_schema=tag_schema, - data_schema=data_schema, - data_context=descriptor.get("data_context_key"), - ) - stored_label = descriptor.get("label") - if stored_label and stored_label != node_name: - node._label = stored_label - else: - raise ValueError( - f"Unknown source_type {source_type!r} in pipeline descriptor." - ) - reconstructed[node_hash] = node - - elif node_type == "function": - up_hashes = upstream_map.get(node_hash, []) - upstream_node = reconstructed.get(up_hashes[0]) if up_hashes else None - node = FunctionNode.from_descriptor( - descriptor, function_pod=None, input_stream=upstream_node, databases={} - ) - reconstructed[node_hash] = node - - elif node_type == "operator": - up_hashes = upstream_map.get(node_hash, []) - upstream_nodes = tuple( - reconstructed[h] for h in up_hashes if h in reconstructed - ) - operator = None - op_config = descriptor.get("operator_config") - if op_config: - try: - from orcapod.pipeline.serialization import resolve_operator_from_config - operator = resolve_operator_from_config(op_config) - except Exception as exc: - logger.warning( - "Could not reconstruct operator %r from config — " - "node will be in read-only mode: %s", - op_config.get("class_name"), - exc, - ) - node = OperatorNode.from_descriptor( - descriptor, operator=operator, input_streams=upstream_nodes, databases={} - ) - reconstructed[node_hash] = node - - # Build Pipeline instance — bypass the recording path entirely. - pipeline = cls(name=name, auto_compile=False) - pipeline._persistent_node_map = dict(reconstructed) - - nodes_by_label: dict[str, GraphNode] = {} - for node in reconstructed.values(): - if node.label: - if node.label not in nodes_by_label: - nodes_by_label[node.label] = node - else: - logger.warning("Label collision in loaded pipeline: %r.", node.label) - pipeline._nodes = nodes_by_label - - pipeline._node_graph = nx.DiGraph() - for up_hash, down_hash in edges: - up_node = reconstructed.get(up_hash) - down_node = reconstructed.get(down_hash) - if up_node is not None and down_node is not None: - pipeline._node_graph.add_edge(up_node, down_node) - for node in reconstructed.values(): - if node not in pipeline._node_graph: - pipeline._node_graph.add_node(node) - - pipeline._graph_edges = [(up, down) for up, down in edges] - pipeline._hash_graph = nx.DiGraph() - for up_hash, down_hash in edges: - pipeline._hash_graph.add_edge(up_hash, down_hash) - for node_hash, node in reconstructed.items(): - if node_hash not in pipeline._hash_graph: - pipeline._hash_graph.add_node(node_hash) - attrs = pipeline._hash_graph.nodes[node_hash] - attrs["node_type"] = node.node_type - if node.label: - attrs["label"] = node.label - - # Populate legacy fields (as produced by compile()) for _build_execution_graph(). - pipeline._node_lut = { - h: n for h, n in reconstructed.items() if n.node_type != "source" - } - pipeline._upstreams = { - h: n for h, n in reconstructed.items() if n.node_type == "source" - } - - pipeline._compiled = True - return pipeline - - def _clone_for_execution(self) -> "Pipeline": - """Create a lightweight copy of this compiled pipeline for isolated execution. - - All structural state is shared read-only with the original. - Only ``_nodes`` gets its own copy so that execution-node label - mutations don't affect other jobs sharing this blueprint. - - Returns: - A new ``Pipeline`` instance sharing read-only state with ``self``. - """ - clone = Pipeline.__new__(Pipeline) - clone._tracker_manager = self._tracker_manager - clone._active = False - clone._name = self._name - clone._invocation_lut = self._invocation_lut - clone._source_streams = self._source_streams - clone._node_lut = self._node_lut - clone._upstreams = self._upstreams - clone._graph_edges = self._graph_edges - clone._hash_graph = self._hash_graph - clone._persistent_node_map = self._persistent_node_map - clone._node_graph = self._node_graph - clone._auto_compile = self._auto_compile - clone._compiled = self._compiled - clone._nodes = dict(self._nodes) # own copy — mutations don't affect original - return clone - - def __dir__(self) -> list[str]: - return list(super().__dir__()) + list(self._nodes.keys()) -``` - -- [ ] **Step 4.2: Run the full existing pipeline test suite** - -```bash -uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: existing tests should mostly pass. New tests in `test_pipeline_base_recording.py` and `test_invocation_transitions.py` should now start passing too (Pipeline no longer has its own compile() competing with the base). - -- [ ] **Step 4.3: Fix any import errors in Pipeline** - -The new `Pipeline` class body references `cp` (the `core_protocols` import) and `AutoRegisteringContextBasedTracker` — both are already imported at the top of `graph.py`. Confirm the imports at the top of `graph.py` still include: - -```python -from orcapod.core.nodes import ( - FunctionNode, - GraphNode, - OperatorNode, - SourceNode, -) -from orcapod.core.tracker import AutoRegisteringContextBasedTracker -from orcapod.pipeline.base import AbstractPipelineBase -from orcapod.protocols import core_protocols as cp -``` - -The node imports (`FunctionNode`, `GraphNode`, `OperatorNode`, `SourceNode`) are still needed by `save()` and `load()`. Leave them. - -- [ ] **Step 4.4: Commit** - -```bash -git add src/orcapod/pipeline/graph.py -git commit -m "refactor(pipeline): Pipeline sheds recording/compile overrides; uses AbstractPipelineBase.compile()" -``` - ---- - -## Task 5: Add `SourceJobNode.from_stream()` three-way logic - -**Files:** -- Modify: `src/orcapod/core/nodes/source_node.py` - -Read `src/orcapod/core/nodes/source_node.py` before editing. Locate the `SourceJobNode` class (around line 395). After the `as_node()` method (around line 511), add the new `from_stream()` classmethod. - -- [ ] **Step 5.1: Run SourceJobNode.from_stream tests to confirm they fail** - -```bash -uv run pytest tests/test_pipeline/test_invocation_transitions.py::TestSourceJobNodeFromStream -v -``` - -Expected: `AttributeError: type object 'SourceJobNode' has no attribute 'from_stream'` - -- [ ] **Step 5.2: Add from_stream() to SourceJobNode** - -In `src/orcapod/core/nodes/source_node.py`, add the following classmethod to `SourceJobNode` immediately after the `as_node()` method (after line 521): - -```python - @classmethod - def from_stream( - cls, - stream: "StreamProtocol", - name: str | None = None, - ) -> "SourceJobNode": - """Create a ``SourceJobNode`` from *stream*, applying three-way logic. - - The input type determines the binding behavior: - - * ``SourceJobNode`` input — copy schema and ``bound_source``; do **not** - wrap the input SJN itself as the bound source (avoids double-wrapping). - * ``SourceNode`` input — create an **unbound** ``SourceJobNode`` preserving - the schema placeholder semantics. The unbound SJN has the same - ``content_hash()`` as the original ``SourceNode`` (both schema-based). - * Any other stream — create a **bound** ``SourceJobNode`` with the stream - as ``bound_source``. The SJN's ``content_hash()`` equals the stream's. - - This three-way logic is critical for the Pipeline → PipelineJob transition: - ``Pipeline.to_invocations()`` produces ``SourceNode``-backed source streams, - and ``PipelineJob.from_invocations()`` must preserve the - concrete/placeholder distinction when calling this method. - - Args: - stream: The upstream stream to wrap. - name: Optional explicit slot name. Defaults to ``stream.label`` for - non-SJN/SourceNode inputs; ``stream.name`` for SJN and SourceNode. - - Returns: - A new ``SourceJobNode`` with binding reflecting the input type. - """ - if isinstance(stream, SourceJobNode): - # SJN → SJN: copy bound_source (do NOT wrap the SJN itself) - return cls( - name=name if name is not None else stream.name, - tag_schema=stream.tag_schema, - data_schema=stream.data_schema, - bound_source=stream.bound_source, - ) - elif isinstance(stream, SourceNode): - # SourceNode → unbound SJN: preserve schema placeholder - return cls( - name=name if name is not None else stream.name, - tag_schema=stream.tag_schema, - data_schema=stream.data_schema, - bound_source=None, - ) - else: - # Concrete stream → bound SJN - tag_schema, data_schema = stream.output_schema() - slot_name = name if name is not None else stream.label - return cls( - name=slot_name, - tag_schema=tag_schema, - data_schema=data_schema, - bound_source=stream, - ) -``` - -- [ ] **Step 5.3: Run SourceJobNode.from_stream tests** - -```bash -uv run pytest tests/test_pipeline/test_invocation_transitions.py::TestSourceJobNodeFromStream -v -``` - -Expected: all 4 tests pass. - -- [ ] **Step 5.4: Commit** - -```bash -git add src/orcapod/core/nodes/source_node.py -git commit -m "feat(source_node): add SourceJobNode.from_stream() with three-way input logic" -``` - ---- - -## Task 6: Update `PipelineJob` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` - -Read `src/orcapod/pipeline/job.py` before editing. This is a large file — make targeted changes described below rather than rewriting the whole class. - -- [ ] **Step 6.1: Update PipelineJob.__init__ — remove _rec_* fields** - -In `__init__`, remove these six lines: -```python - # Recording state (populated during with-block) - self._rec_graph_edges: list[tuple[str, str]] = [] - self._rec_upstreams: dict[str, cp.StreamProtocol] = {} - self._rec_node_lut: dict[str, "GraphNode"] = {} - self._spec_by_name: dict[str, "SourceNode"] = {} - self._unresolved_specs: list[str] = [] -``` - -Keep `_unresolved_specs` — it is used by `run()` and `unresolved_specs` property. Only remove the `_rec_*` and `_spec_by_name` lines. The remaining `__init__` body should look like: - -```python - def __init__( - self, - name: str | tuple[str, ...] = "pipeline", - store: "ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, - tracker_manager: cp.TrackerManagerProtocol | None = None, - *, - _pipeline: "Pipeline | None" = None, - sources: "dict[str, cp.StreamProtocol] | None" = None, - ) -> None: - super().__init__(name=name, tracker_manager=tracker_manager) - self._store = store - self._execution_context = execution_context - self._compiled_pipeline: "Pipeline | None" = _pipeline - self._sources: dict[str, cp.StreamProtocol] = dict(sources or {}) - self._unresolved_specs: list[str] = [] - self._has_run: bool = False - self._run_id: str | None = None - - # Job-node map initialised by AbstractPipelineBase.__init__; _compiled_pipeline - # is set above for jobs created via from_pipeline() / load(). - self._persistent_node_map: "dict[str, Any] | None" = None - self._nodes: "dict[str, Any]" = {} -``` - -- [ ] **Step 6.2: Add node-factory class properties and thin compile() override** - -Add these after the `__init__` method, replacing the `__enter__` override (the whole `__enter__` method can be deleted — `super().__enter__()` is the only line that matters and it's inherited): - -```python - # ------------------------------------------------------------------ - # Node-factory class attributes (used by AbstractPipelineBase.compile()) - # ------------------------------------------------------------------ - - @property - def source_node_class(self) -> type: - from orcapod.core.nodes.source_node import SourceJobNode - return SourceJobNode - - @property - def function_node_class(self) -> type: - from orcapod.core.nodes.function_node import FunctionJobNode - return FunctionJobNode - - @property - def operator_node_class(self) -> type: - from orcapod.core.nodes.operator_node import OperatorJobNode - return OperatorJobNode - - # ------------------------------------------------------------------ - # Context manager — recording - # ------------------------------------------------------------------ - - def __enter__(self) -> "PipelineJob": - return super().__enter__() # type: ignore[return-value] - - # ------------------------------------------------------------------ - # compile() — thin override: runs base compile then resets pipeline cache - # ------------------------------------------------------------------ - - def compile(self) -> None: - """Compile recorded invocations using the base class pass, then reset pipeline cache. - - The base ``compile()`` builds ``_persistent_node_map`` using - ``SourceJobNode`` / ``FunctionJobNode`` / ``OperatorJobNode`` class - properties. This override resets ``_compiled_pipeline`` so that the - lazily-computed ``compiled_pipeline`` property rebuilds it fresh on - next access. - - Also wires databases to all job nodes if a store is already attached. - """ - super().compile() - self._compiled_pipeline = None # reset cache; rebuilt lazily via compiled_pipeline - if self._store is not None: - self._distribute_databases() -``` - -- [ ] **Step 6.3: Replace the old compile() method** - -Remove the existing `compile()` method body (lines 99–211 in the original file — the large method that built a Pipeline internally and then walked it). It is replaced by the thin override added in Step 6.2. Also remove `_ensure_source_node()` (the whole method), `_is_concrete_source()` (the whole staticmethod), `_to_node_stream()` (the whole method), and the two `record_*` override methods. These are all replaced by the inherited base-class implementations. - -- [ ] **Step 6.4: Add compiled_pipeline lazy property; update pipeline property and _build_execution_graph()** - -Replace the existing `pipeline` property: - -```python - @property - def pipeline(self) -> "Pipeline": - """The compiled pure Pipeline (SourceNode-only leaves). - - Raises: - RuntimeError: If no DAG has been recorded yet. - """ - return self.compiled_pipeline - - @property - def compiled_pipeline(self) -> "Pipeline": - """Lazily compute and cache the pure Pipeline blueprint for this job. - - Computed via ``as_pipeline()`` on first access after a compile. - - Raises: - RuntimeError: If no DAG has been recorded or loaded yet. - """ - if self._compiled_pipeline is None: - if not self._compiled: - raise RuntimeError( - "PipelineJob has no compiled pipeline yet. " - "Use 'with job:' to record a DAG first." - ) - self._compiled_pipeline = self.as_pipeline() - return self._compiled_pipeline -``` - -In `_build_execution_graph()`, change the first assignment inside the method from: -```python - pipeline = self._compiled_pipeline - if pipeline is None: - raise RuntimeError("No compiled pipeline — use 'with job:' first.") -``` -to: -```python - pipeline = self.compiled_pipeline -``` - -(The `compiled_pipeline` property raises `RuntimeError` when appropriate, so the explicit check can be removed.) - -- [ ] **Step 6.5: Replace from_pipeline() with thin composition** - -Replace the entire `from_pipeline()` classmethod (which currently has ~80 lines of topological walk logic) with: - -```python - @classmethod - def from_pipeline( - cls, - pipeline: "Pipeline", - store: "ArrowDatabaseProtocol | None" = None, - sources: "dict[str, cp.StreamProtocol] | None" = None, - execution_context: "ExecutionContext | None" = None, - ) -> "PipelineJob": - """Create a runnable ``PipelineJob`` from a compiled ``Pipeline``. - - Converts *pipeline* to an ``InvocationGraph`` via ``to_invocations()``, - then reconstructs a ``PipelineJob`` via ``from_invocations()``. Source - bindings and store are applied via ``bind()`` after construction. - - Args: - pipeline: A compiled ``Pipeline`` (``pipeline._compiled`` must be ``True``). - store: Database for result caching and operator records. - sources: Mapping of ``SourceNode.name`` → concrete source. - execution_context: Optional execution configuration. - - Returns: - A new ``PipelineJob`` ready to run (or ``bind()`` further). - - Raises: - ValueError: If *pipeline* has not been compiled. - """ - if not pipeline._compiled: - raise ValueError( - "Pipeline must be compiled before creating a PipelineJob from it. " - "Call pipeline.compile() or use auto_compile=True." - ) - job: "PipelineJob" = cls.from_invocations(pipeline.to_invocations(), name=pipeline.name) - job._store = store - job._execution_context = execution_context - if sources or store: - job.bind(sources=sources, store=store) - return job -``` - -- [ ] **Step 6.6: Replace as_pipeline() with thin composition** - -Replace the entire `as_pipeline()` method body (which currently has ~70 lines of topological walk building FunctionNode/OperatorNode) with: - -```python - def as_pipeline(self) -> "Pipeline": - """Return the lightweight ``Pipeline`` blueprint for this job. - - Converts this job's ``_persistent_node_map`` to an ``InvocationGraph`` - via ``to_invocations()``, then reconstructs a ``Pipeline`` via - ``Pipeline.from_invocations()``. This ensures the returned Pipeline's - node hashes match those of this job. - - Returns: - A compiled ``Pipeline`` whose ``_persistent_node_map`` contains - only ``SourceNode`` / ``FunctionNode`` / ``OperatorNode`` objects. - - Raises: - RuntimeError: If this job has not been compiled. - """ - from orcapod.pipeline.graph import Pipeline - - if not self._compiled: - raise RuntimeError( - "PipelineJob has not been compiled. " - "Either use 'with job:' to record a DAG, " - "or create the job via PipelineJob.from_pipeline()." - ) - return Pipeline.from_invocations(self.to_invocations(), name=self._name) -``` - -- [ ] **Step 6.7: Update from_pipeline() call in load() if needed** - -Verify that `PipelineJob.load()` does not call the old `from_pipeline()` with positional args that are now changed. Search the `load()` method for `from_pipeline` — it should not be called there; `load()` constructs the job directly via `cls(...)`. No change needed. - -- [ ] **Step 6.8: Run transition tests** - -```bash -uv run pytest tests/test_pipeline/test_invocation_transitions.py -v -``` - -Expected: all pass. - -- [ ] **Step 6.9: Commit** - -```bash -git add src/orcapod/pipeline/job.py -git commit -m "refactor(pipeline): PipelineJob sheds _rec_* fields; from_pipeline/as_pipeline become thin compositions" -``` - ---- - -## Task 7: Full test suite — fix regressions - -- [ ] **Step 7.1: Run the full pipeline test suite** - -```bash -uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -60 -``` - -Fix any failures before proceeding. Common failure modes: -- `AttributeError: _rec_node_lut` — a test accesses the removed field; update test to use `_invocation_lut` or `_persistent_node_map` instead -- `AttributeError: _rec_graph_edges` — same; use `_graph_edges` (now a compiled artifact) or `_hash_graph` -- `AttributeError: _spec_by_name` — update test to inspect `_sources` instead -- Failures in `test_serialization.py` — Pipeline's `save()` / `load()` round-trip; run those in isolation and verify edge serialization still works via `_graph_edges` (now populated from `_hash_graph.edges()` at end of `compile()`) - -- [ ] **Step 7.2: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short -q 2>&1 | tail -40 -``` - -All tests should pass. - -- [ ] **Step 7.3: Commit all fixes** - -```bash -git add -p # stage only fix changes -git commit -m "fix(pipeline): fix regressions after Pipeline/PipelineJob recording refactor" -``` - ---- - -## Self-Review - -**Spec coverage check:** - -| Spec requirement | Covered by | -|---|---| -| `PodInvocation` hierarchy (`FunctionInvocation`, `OperatorInvocation`) | Task 2 | -| `AbstractPipelineBase`: concrete recording methods, `_record_invocation` | Task 3 | -| `AbstractPipelineBase`: class-property factories (abstract) | Task 3 | -| `AbstractPipelineBase`: `InvocationGraph`, `from_invocations`, `to_invocations` | Task 3 | -| `AbstractPipelineBase`: unified `compile()` | Task 3 | -| `AbstractPipelineBase`: `_node_lut`/`_upstreams`/`_graph_edges` additive semantics updated | Task 3 | -| `AbstractPipelineBase`: `reset()` → no-op | Task 3 | -| `Pipeline`: remove overrides, add class properties | Task 4 | -| `SourceJobNode.from_stream()` three-way logic | Task 5 | -| `PipelineJob`: remove `_rec_*` fields, add class properties | Task 6 | -| `PipelineJob`: thin `from_pipeline()`/`as_pipeline()` | Task 6 | -| `PipelineJob`: lazy `compiled_pipeline` property | Task 6 | -| Tests-first | Task 1 (all test files written before production code) | -| `SourceJobNode.identity_structure()` bound-state dependent — already implemented, must not change | Verified in Task 5 (no changes to `identity_structure`) | - -**Backward compat notes:** -- `Pipeline.save()` / `load()` — unchanged; `_graph_edges` is now populated from `_hash_graph.edges()` at end of `compile()` so save still works. -- `_build_execution_graph()` — one-line change (`self.compiled_pipeline` instead of `self._compiled_pipeline`) preserves its behavior via lazy property. -- `Pipeline._clone_for_execution()` — updated in Task 4 to copy `_invocation_lut` and `_source_streams` alongside the other shared fields. diff --git a/superpowers/plans/2026-05-23-rename-config-to-orcapodconfig.md b/superpowers/plans/2026-05-23-rename-config-to-orcapodconfig.md deleted file mode 100644 index b78eda2e2..000000000 --- a/superpowers/plans/2026-05-23-rename-config-to-orcapodconfig.md +++ /dev/null @@ -1,542 +0,0 @@ -# Rename Config → OrcapodConfig Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Rename the `Config` class in `orcapod/config.py` to `OrcapodConfig` everywhere it appears — class definition, all internal usages, all test usages — and add it to the top-level public API. - -**Architecture:** Pure identifier rename with no logic changes. `config.py` is the single source of truth; all 15 consumer source files and 2 test files reference the class by import. The top-level `__init__.py` does not currently expose `Config`, but will expose `OrcapodConfig` after this rename. No backward-compat alias — project CLAUDE.md prohibits shims pre-v0.1.0. - -**Tech Stack:** Python 3.12+, `uv run` for all commands, `pytest` for tests. - ---- - -## File Map - -| Action | File | -|---|---| -| Modify | `src/orcapod/config.py` — rename class + update internal refs | -| Modify | `src/orcapod/__init__.py` — add OrcapodConfig to public API | -| Modify | `src/orcapod/core/base.py` — import + 3 type hints | -| Modify | `src/orcapod/core/datagrams/datagram.py` — import + 1 type hint | -| Modify | `src/orcapod/core/data_function.py` — import + 2 type hints | -| Modify | `src/orcapod/core/function_pod.py` — import + 1 type hint | -| Modify | `src/orcapod/core/nodes/function_node.py` — import + 3 type hints | -| Modify | `src/orcapod/core/nodes/operator_node.py` — import + 2 type hints | -| Modify | `src/orcapod/core/nodes/source_node.py` — import + 1 type hint | -| Modify | `src/orcapod/core/operators/static_output_pod.py` — import + 1 type hint | -| Modify | `src/orcapod/core/sources/base.py` — import + 1 type hint | -| Modify | `src/orcapod/core/sources/cached_source.py` — import + 1 type hint | -| Modify | `src/orcapod/core/sources/db_table_source.py` — TYPE_CHECKING import + 1 type hint | -| Modify | `src/orcapod/core/sources/postgresql_table_source.py` — TYPE_CHECKING import + 1 type hint | -| Modify | `src/orcapod/core/sources/spiraldb_table_source.py` — TYPE_CHECKING import + 1 type hint | -| Modify | `src/orcapod/core/sources/sqlite_table_source.py` — TYPE_CHECKING import + 1 type hint | -| Modify | `src/orcapod/core/sources/stream_builder.py` — TYPE_CHECKING import + 1 type hint | -| Modify | `tests/test_core/operators/test_operators.py` — 2 local imports + usages | -| Modify | `tests/test_core/operators/test_merge_join.py` — 1 local import + usage | -| Create | `tests/test_orcapod_config.py` — new tests for OrcapodConfig name | - ---- - -### Task 1: Write the Failing Test - -**Files:** -- Create: `tests/test_orcapod_config.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_orcapod_config.py -"""Tests for OrcapodConfig class naming (ENG-514).""" - - -class TestOrcapodConfigModule: - def test_can_import_orcapod_config_from_config_module(self): - from orcapod.config import OrcapodConfig # noqa: F401 - - def test_orcapod_config_is_instantiable_with_defaults(self): - from orcapod.config import OrcapodConfig - - cfg = OrcapodConfig() - assert cfg.system_tag_hash_n_char == 12 - assert cfg.schema_hash_n_char == 12 - assert cfg.path_hash_n_char == 20 - - def test_default_config_is_orcapod_config_instance(self): - from orcapod.config import DEFAULT_CONFIG, OrcapodConfig - - assert isinstance(DEFAULT_CONFIG, OrcapodConfig) - - def test_orcapod_config_with_updates(self): - from orcapod.config import OrcapodConfig - - cfg = OrcapodConfig() - updated = cfg.with_updates(system_tag_hash_n_char=8) - assert updated.system_tag_hash_n_char == 8 - assert cfg.system_tag_hash_n_char == 12 # original unchanged - - def test_orcapod_config_merge(self): - from orcapod.config import OrcapodConfig - - base = OrcapodConfig() - other = OrcapodConfig(system_tag_hash_n_char=8) - merged = base.merge(other) - assert merged.system_tag_hash_n_char == 8 - assert merged.schema_hash_n_char == 12 # unchanged default - - def test_orcapod_config_merge_type_error(self): - import pytest - - from orcapod.config import OrcapodConfig - - cfg = OrcapodConfig() - with pytest.raises(TypeError): - cfg.merge("not a config") # type: ignore[arg-type] - - -class TestOrcapodConfigTopLevelExport: - def test_can_import_orcapod_config_from_orcapod(self): - from orcapod import OrcapodConfig # noqa: F401 - - def test_orcapod_config_in_all(self): - import orcapod - - assert "OrcapodConfig" in orcapod.__all__ -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -cd /path/to/orcapod-python # replace with actual repo path -uv run pytest tests/test_orcapod_config.py -v -``` - -Expected: FAIL — `ImportError: cannot import name 'OrcapodConfig' from 'orcapod.config'` - ---- - -### Task 2: Rename Class in config.py - -**Files:** -- Modify: `src/orcapod/config.py` - -- [ ] **Step 1: Replace the entire file with the renamed version** - -Replace the full contents of `src/orcapod/config.py` with: - -```python -# config.py -from dataclasses import dataclass, replace -from typing import Self - - -@dataclass(frozen=True) -class OrcapodConfig: - """Immutable OrcaPod configuration object.""" - - system_tag_hash_n_char: int = 12 - schema_hash_n_char: int = 12 - path_hash_n_char: int = 20 - - def with_updates(self, **kwargs) -> Self: - """Create a new ``OrcapodConfig`` instance with updated values.""" - return replace(self, **kwargs) - - def merge(self, other: "OrcapodConfig") -> "OrcapodConfig": - """Merge with another config, other takes precedence.""" - if not isinstance(other, OrcapodConfig): - raise TypeError("Can only merge with another OrcapodConfig instance") - - # Get all non-default values from other - defaults = OrcapodConfig() - updates = {} - for field_name in self.__dataclass_fields__: - other_value = getattr(other, field_name) - default_value = getattr(defaults, field_name) - if other_value != default_value: - updates[field_name] = other_value - - return self.with_updates(**updates) - - -# Module-level default config - created at import time -DEFAULT_CONFIG = OrcapodConfig() -``` - -- [ ] **Step 2: Run the new tests to verify they pass (except the top-level export tests)** - -```bash -uv run pytest tests/test_orcapod_config.py::TestOrcapodConfigModule -v -``` - -Expected: All 6 tests in `TestOrcapodConfigModule` PASS. -`TestOrcapodConfigTopLevelExport` will still fail — that is expected until Task 4. - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/config.py tests/test_orcapod_config.py -git commit -m "refactor(config): rename Config → OrcapodConfig" -``` - ---- - -### Task 3: Update All Consumer Source and Test Files - -**Files:** -- Modify: all 15 source files that import `Config` -- Modify: 2 test files that locally import `Config` - -Every file that imports `Config` from `orcapod.config` needs two kinds of edits: -1. The import line: `Config` → `OrcapodConfig` -2. Every type hint or usage: `Config` → `OrcapodConfig` - -`ColumnConfig`, `NodeConfig`, `PipelineConfig` are distinct classes and must **not** be changed. -The `sed` pattern `\bConfig\b` (whole-word match) safely renames only the bare word `Config`. - -- [ ] **Step 1: Update the 10 source files with direct (non-TYPE_CHECKING) imports** - -For each file listed below, make the two edits shown. - -**`src/orcapod/core/base.py`** — import line 9, type hints lines 78, 88, 120, 363: - -``` -Old: from orcapod.config import DEFAULT_CONFIG, Config -New: from orcapod.config import DEFAULT_CONFIG, OrcapodConfig -``` -Then replace every remaining `Config` (type hint only — `ColumnConfig`, `NodeConfig` are not present in this file): -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (lines 78, 120, 363) -- `def orcapod_config(self) -> Config:` → `def orcapod_config(self) -> OrcapodConfig:` (line 88) - -**`src/orcapod/core/datagrams/datagram.py`** — import line 29, type hint line 71: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 71) - -**`src/orcapod/core/data_function.py`** — import line 14, type hints lines 138, 377: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (lines 138, 377) - -**`src/orcapod/core/function_pod.py`** — import line 12, type hint line 68: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 68) - -**`src/orcapod/core/nodes/function_node.py`** — import line 26, type hints lines 99, 453, 667: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (lines 99, 453, 667) - -**`src/orcapod/core/nodes/operator_node.py`** — import line 26, type hints lines 77, 520: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (lines 77, 520) - -**`src/orcapod/core/nodes/source_node.py`** — import line 15, type hint line 58: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 58) - -**`src/orcapod/core/operators/static_output_pod.py`** — import line 11, type hint line 272: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 272) - -**`src/orcapod/core/sources/base.py`** — import line 8, type hint line 63: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 63) - -**`src/orcapod/core/sources/cached_source.py`** — import line 8, type hint line 62: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 62) - -- [ ] **Step 2: Update the 5 TYPE_CHECKING source files** - -These files import `Config` inside `if TYPE_CHECKING:` blocks. Same two edits each. - -**`src/orcapod/core/sources/db_table_source.py`** — TYPE_CHECKING import line 26, type hint line 72: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 72) - -**`src/orcapod/core/sources/postgresql_table_source.py`** — TYPE_CHECKING import line 22, type hint line 72: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config | None = None,` → `config: OrcapodConfig | None = None,` (line 72) - -**`src/orcapod/core/sources/spiraldb_table_source.py`** — TYPE_CHECKING import line 45, type hint line 101: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: "Config | None" = None,` → `config: "OrcapodConfig | None" = None,` (line 101) - -**`src/orcapod/core/sources/sqlite_table_source.py`** — TYPE_CHECKING import line 32, type hint line 82: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: "Config | None" = None,` → `config: "OrcapodConfig | None" = None,` (line 82) - -**`src/orcapod/core/sources/stream_builder.py`** — TYPE_CHECKING import line 24, type hint line 62: - -``` -Old: from orcapod.config import Config -New: from orcapod.config import OrcapodConfig -``` -- `config: Config` → `config: OrcapodConfig` (line 62) - -- [ ] **Step 3: Update the 2 test files with local Config imports** - -**`tests/test_core/operators/test_operators.py`** — 2 local imports at lines 1316 and 1411: - -At line 1316 (inside a test function body): -``` -Old: from orcapod.config import Config - ... - n_char = Config().system_tag_hash_n_char -New: from orcapod.config import OrcapodConfig - ... - n_char = OrcapodConfig().system_tag_hash_n_char -``` - -At line 1411 (inside a second test function body): -``` -Old: from orcapod.config import Config - ... - n_char = Config().system_tag_hash_n_char -New: from orcapod.config import OrcapodConfig - ... - n_char = OrcapodConfig().system_tag_hash_n_char -``` - -**`tests/test_core/operators/test_merge_join.py`** — 1 local import at line 644: - -``` -Old: from orcapod.config import Config - ... - n_char = Config().system_tag_hash_n_char -New: from orcapod.config import OrcapodConfig - ... - n_char = OrcapodConfig().system_tag_hash_n_char -``` - -- [ ] **Step 4: Run the full test suite to verify no regressions** - -```bash -uv run pytest tests/ -v --ignore=tests/test_orcapod_config.py::TestOrcapodConfigTopLevelExport -x -``` - -Expected: All existing tests PASS. The `TestOrcapodConfigTopLevelExport` class in `test_orcapod_config.py` will still fail — skip or ignore it for now; it passes after Task 4. - -- [ ] **Step 5: Commit** - -```bash -git add \ - src/orcapod/core/base.py \ - src/orcapod/core/datagrams/datagram.py \ - src/orcapod/core/data_function.py \ - src/orcapod/core/function_pod.py \ - src/orcapod/core/nodes/function_node.py \ - src/orcapod/core/nodes/operator_node.py \ - src/orcapod/core/nodes/source_node.py \ - src/orcapod/core/operators/static_output_pod.py \ - src/orcapod/core/sources/base.py \ - src/orcapod/core/sources/cached_source.py \ - src/orcapod/core/sources/db_table_source.py \ - src/orcapod/core/sources/postgresql_table_source.py \ - src/orcapod/core/sources/spiraldb_table_source.py \ - src/orcapod/core/sources/sqlite_table_source.py \ - src/orcapod/core/sources/stream_builder.py \ - tests/test_core/operators/test_operators.py \ - tests/test_core/operators/test_merge_join.py -git commit -m "refactor(config): update all Config → OrcapodConfig references" -``` - ---- - -### Task 4: Add OrcapodConfig to Top-Level __init__.py - -**Files:** -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Add the import and __all__ entry** - -Current `src/orcapod/__init__.py`: - -```python -from .core.function_pod import ( - FunctionPod, - function_pod, -) -from .core.nodes.source_node import SourceNode -from .pipeline import Pipeline, PipelineJob - -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -__all__ = [ - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceNode", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", -] -``` - -New `src/orcapod/__init__.py` — add `OrcapodConfig` import after the existing imports and add it to `__all__`: - -```python -from .config import OrcapodConfig -from .core.function_pod import ( - FunctionPod, - function_pod, -) -from .core.nodes.source_node import SourceNode -from .pipeline import Pipeline, PipelineJob - -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -__all__ = [ - "OrcapodConfig", - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceNode", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", -] -``` - -- [ ] **Step 2: Run the full test suite including top-level export tests** - -```bash -uv run pytest tests/ -v -x -``` - -Expected: ALL tests PASS, including `TestOrcapodConfigTopLevelExport`. - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/__init__.py -git commit -m "feat(config): export OrcapodConfig from top-level orcapod package" -``` - ---- - -### Task 5: Final Verification - -- [ ] **Step 1: Run the full test suite one final time** - -```bash -uv run pytest tests/ -v -``` - -Expected: All tests PASS with zero failures. - -- [ ] **Step 2: Verify the public API by hand** - -```bash -uv run python -c " -from orcapod import OrcapodConfig -from orcapod.config import OrcapodConfig, DEFAULT_CONFIG -cfg = OrcapodConfig(system_tag_hash_n_char=8) -print('OrcapodConfig:', cfg) -print('DEFAULT_CONFIG:', DEFAULT_CONFIG) -print('isinstance check:', isinstance(DEFAULT_CONFIG, OrcapodConfig)) -" -``` - -Expected output: -``` -OrcapodConfig: OrcapodConfig(system_tag_hash_n_char=8, schema_hash_n_char=12, path_hash_n_char=20) -DEFAULT_CONFIG: OrcapodConfig(system_tag_hash_n_char=12, schema_hash_n_char=12, path_hash_n_char=20) -isinstance check: True -``` - -- [ ] **Step 3: Confirm no stale `Config` references remain in source or tests** - -```bash -grep -rn '\bConfig\b' src/orcapod/ tests/ \ - --include="*.py" \ - | grep -v "ColumnConfig\|NodeConfig\|PipelineConfig\|CacheConfig\|DatabaseConfig\|# config\|_config\b" -``` - -Expected: Zero lines output. If any lines appear, fix them before creating the PR. - ---- - -## Self-Review Checklist - -- **Spec coverage:** - - ✅ Class definition site renamed (`config.py`) - - ✅ All internal usages — imports, type hints, isinstance checks, `__init__` defaults, factory functions (`config.py` `merge()` method, `DEFAULT_CONFIG`) - - ✅ Public re-export — `OrcapodConfig` added to top-level `orcapod.__init__` - - ✅ Tests and fixtures updated (`test_operators.py`, `test_merge_join.py`) - - ✅ New tests written (`test_orcapod_config.py`) - - ✅ No backward-compat alias (per project CLAUDE.md — greenfield pre-v0.1.0) - - ✅ All tests pass - -- **Out of scope (do NOT touch):** - - `ColumnConfig`, `NodeConfig`, `PipelineConfig` — distinct classes, unchanged - - Any restructuring of fields, defaults, or behavior of OrcapodConfig diff --git a/superpowers/plans/2026-05-26-eng-517-pipeline-job-execution-graph-cleanup.md b/superpowers/plans/2026-05-26-eng-517-pipeline-job-execution-graph-cleanup.md deleted file mode 100644 index eb3124a5f..000000000 --- a/superpowers/plans/2026-05-26-eng-517-pipeline-job-execution-graph-cleanup.md +++ /dev/null @@ -1,1202 +0,0 @@ -# ENG-517: PipelineJob Execution Graph Cleanup — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the copy-on-run pattern from `PipelineJob.run()`, wire `GraphProtocol`/`OrcaDAG` into the orchestrator boundary, consolidate the `unbound_sources` API, and rename `compiled_nodes` → `nodes`. - -**Architecture:** `run()` builds a filtered `OrcaDAG[JobNode]` directly from `_persistent_node_map` (no node cloning), hands it to the orchestrator, then mutates `self._has_run`/`_run_id` in place and returns `self`. Orchestrators are updated to accept `GraphProtocol[JobNode]` instead of `nx.DiGraph`. `unbound_source_nodes()` and `_unresolved_specs`/`unresolved_specs` are removed; a single live-computed `unbound_sources` property replaces both. `compiled_nodes` is renamed `nodes` on `AbstractPipelineBase` — the `compiled` qualifier is redundant post-ENG-515 since nodes are always compiled before use. - -**Tech Stack:** Python 3.11+, `uv run pytest` for all test runs, `graphlib.TopologicalSorter` (stdlib) inside `OrcaDAG`. - ---- - -## File Map - -| File | Change | -|---|---| -| `src/orcapod/pipeline/dag.py` | Rename `GraphBackend` → `GraphProtocol`; add `ancestors()` to protocol + `OrcaDAG` | -| `src/orcapod/pipeline/networkx_backend.py` | Update docstring reference; add `ancestors()` | -| `src/orcapod/pipeline/base.py` | Rename `compiled_nodes` property → `nodes` | -| `src/orcapod/pipeline/job.py` | Add `unbound_sources`; remove `unbound_source_nodes`, `_unresolved_specs`, `unresolved_specs`; rename `spec_names`→`source_names`; update `save`/`load`; rewrite `run()`; delete `_build_execution_graph()` / `build_execution_graph()` | -| `src/orcapod/pipeline/sync_orchestrator.py` | Accept `GraphProtocol`, drop `nx.topological_sort` | -| `src/orcapod/pipeline/async_orchestrator.py` | Accept `GraphProtocol`, drop `nx.topological_sort` | -| `tests/test_pipeline/test_dag.py` | Add `ancestors()` tests | -| `tests/test_pipeline/test_networkx_backend.py` | Update `GraphBackend` → `GraphProtocol`; add `ancestors()` test | -| `tests/test_pipeline/test_pipeline_job.py` | Migrate `unbound_source_nodes` → `unbound_sources`; `unresolved_specs` → `unbound_sources`; `compiled_nodes` → `nodes`; rewrite `test_run_is_non_mutating` | -| `tests/test_pipeline/test_pipeline.py` | `compiled_nodes` → `nodes` throughout | -| `tests/test_pipeline/test_serialization.py` | `compiled_nodes` → `nodes`; `unresolved_specs` → `unbound_sources` | -| `tests/test_channels/test_pipeline_async_integration.py` | `compiled_nodes` → `nodes` throughout | -| `tests/test_pipeline/test_integration_smoke.py` | `compiled_nodes` → `nodes` | -| `tests/test_pipeline/test_orchestrator_executor_matrix.py` | `compiled_nodes` → `nodes` | -| `tests/test_pipeline/test_sync_orchestrator.py` | `compiled_nodes` → `nodes` | -| `tests/test_data/test_polars_nullability/test_function_node_nullability.py` | `compiled_nodes` → `nodes` | - ---- - -## Task 1: Create the feature branch - -**Files:** none - -- [ ] **Step 1: Check out the branch** - -```bash -cd /home/kurouto/kurouto-jobs/71dc4f00-85e9-4767-a435-8d1f5e163103/orcapod-python -git checkout -b eywalker/eng-517-review-cleanup-of-pipelinejob-execution-graph-remove -git branch --show-current -``` - -Expected output: `eywalker/eng-517-review-cleanup-of-pipelinejob-execution-graph-remove` - ---- - -## Task 2: Rename `compiled_nodes` → `nodes` on `AbstractPipelineBase` - -**Files:** -- Modify: `src/orcapod/pipeline/base.py` -- Modify: `tests/test_pipeline/test_pipeline_job.py` -- Modify: `tests/test_pipeline/test_pipeline.py` -- Modify: `tests/test_pipeline/test_serialization.py` -- Modify: `tests/test_channels/test_pipeline_async_integration.py` -- Modify: `tests/test_pipeline/test_integration_smoke.py` -- Modify: `tests/test_pipeline/test_orchestrator_executor_matrix.py` -- Modify: `tests/test_pipeline/test_sync_orchestrator.py` -- Modify: `tests/test_data/test_polars_nullability/test_function_node_nullability.py` - -- [ ] **Step 1: Rename the property in `base.py` (line 115)** - -In `src/orcapod/pipeline/base.py`, replace: - -```python - @property - def compiled_nodes(self) -> dict[str, Any]: - """Copy of the compiled nodes dict (label → node).""" - return self._nodes.copy() -``` - -With: - -```python - @property - def nodes(self) -> dict[str, Any]: - """Copy of the compiled nodes dict (label → node).""" - return self._nodes.copy() -``` - -- [ ] **Step 2: Bulk-rename all call sites** - -Run this sed command to replace every occurrence across the codebase (source + tests): - -```bash -cd /home/kurouto/kurouto-jobs/71dc4f00-85e9-4767-a435-8d1f5e163103/orcapod-python -grep -rn "compiled_nodes" --include="*.py" -l -``` - -Then for each file listed, replace `compiled_nodes` with `nodes`. Use the project's grep/sed tooling or do it file-by-file. The full list from the codebase: - -- `tests/test_channels/test_pipeline_async_integration.py` — 11 occurrences -- `tests/test_pipeline/test_pipeline.py` — ~35 occurrences -- `tests/test_pipeline/test_pipeline_job.py` — ~12 occurrences -- `tests/test_pipeline/test_serialization.py` — 2 occurrences -- `tests/test_pipeline/test_integration_smoke.py` — 2 occurrences -- `tests/test_pipeline/test_orchestrator_executor_matrix.py` — 1 occurrence -- `tests/test_pipeline/test_sync_orchestrator.py` — 1 occurrence -- `tests/test_data/test_polars_nullability/test_function_node_nullability.py` — 1 occurrence -- `src/orcapod/pipeline/job.py` — docstrings only (2 occurrences) - -Apply the rename with: - -```bash -find . -name "*.py" | xargs sed -i 's/compiled_nodes/nodes/g' -``` - -- [ ] **Step 3: Also rename test methods that mention `compiled_nodes` in their names** - -In `tests/test_pipeline/test_pipeline.py`: -- `test_source_nodes_in_compiled_nodes` → `test_source_nodes_in_nodes` -- Any other method name containing `compiled_nodes` - -In `tests/test_pipeline/test_integration_smoke.py`: -- `test_smoke_compiled_nodes_accessible_after_run` → `test_smoke_nodes_accessible_after_run` - -Search for them: -```bash -grep -rn "def test.*compiled_nodes" --include="*.py" -``` - -Rename each found method. - -- [ ] **Step 4: Verify no `compiled_nodes` remains except in git history** - -```bash -grep -rn "compiled_nodes" --include="*.py" . -``` - -Expected: zero results. - -- [ ] **Step 5: Run tests to confirm rename is clean** - -```bash -uv run pytest tests/ -v 2>&1 | tail -20 -``` - -Expected: all pass. - -- [ ] **Step 6: Commit** - -```bash -git add -u -git commit -m "refactor(pipeline): rename compiled_nodes → nodes on AbstractPipelineBase" -``` - ---- - -## Task 3: Rename `GraphBackend` → `GraphProtocol` and add `ancestors()` to `OrcaDAG` - - -**Files:** -- Modify: `src/orcapod/pipeline/dag.py` (lines 33, 59–95, 348) -- Modify: `tests/test_pipeline/test_dag.py` -- Modify: `tests/test_pipeline/test_networkx_backend.py` (lines 4–5, 14, 23, 26, 30, 33, 35) - -- [ ] **Step 1: Write failing tests for `ancestors()`** - -Add to the bottom of `tests/test_pipeline/test_dag.py`: - -```python -class TestOrcaDAGAncestors: - def test_ancestors_of_source_node_is_empty(self): - """A source node (no predecessors) has no ancestors.""" - dag = OrcaDAG() - dag.add_node("a") - assert dag.ancestors("a") == frozenset() - - def test_ancestors_returns_direct_predecessor(self): - dag = OrcaDAG() - dag.add_edge("a", "b") - assert dag.ancestors("b") == frozenset({"a"}) - - def test_ancestors_returns_transitive_predecessors(self): - """ancestors() walks all the way back to source nodes.""" - dag = OrcaDAG() - dag.add_edge("a", "b") - dag.add_edge("b", "c") - assert dag.ancestors("c") == frozenset({"a", "b"}) - assert dag.ancestors("b") == frozenset({"a"}) - - def test_ancestors_handles_diamond(self): - """Two paths to same ancestor — no duplicates.""" - dag = OrcaDAG() - dag.add_edge("a", "b") - dag.add_edge("a", "c") - dag.add_edge("b", "d") - dag.add_edge("c", "d") - assert dag.ancestors("d") == frozenset({"a", "b", "c"}) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestOrcaDAGAncestors -v -``` - -Expected: `AttributeError: 'OrcaDAG' object has no attribute 'ancestors'` - -- [ ] **Step 3: Rename `GraphBackend` → `GraphProtocol` and add `ancestors()` in `dag.py`** - -In `src/orcapod/pipeline/dag.py`: - -**Line 33** — update `__all__`: -```python -__all__ = ["Comparable", "GraphProtocol", "OrcaDAG", "CycleError"] -``` - -**Lines 58–95** — rename class and add `ancestors()` stub at end of protocol: -```python -@runtime_checkable -class GraphProtocol(Protocol[NodeT]): - """Structural protocol for DAG backend implementations. - - Both ``OrcaDAG`` and ``NetworkxBackend`` satisfy this protocol, enabling - callers to switch graph implementations via a config flag (ENG-494). - - NodeT must be ``Hashable``. The ``topological_sort_deterministic`` method - is not part of this protocol because it additionally requires NodeT to - satisfy ``Comparable``; callers that need deterministic ordering should - use ``OrcaDAG`` or ``NetworkxBackend`` directly with a comparable node type - (e.g. ``OrcaDAG[str]``). - """ - - def add_node(self, node: NodeT, **attrs: Any) -> None: ... - - def add_edge(self, u: NodeT, v: NodeT) -> None: ... - - def node_attrs(self, node: NodeT) -> dict[str, Any]: ... - - def __contains__(self, node: object) -> bool: ... - - def __len__(self) -> int: ... - - def __iter__(self) -> Iterator[NodeT]: ... - - def nodes(self) -> Iterable[NodeT]: ... - - def edges(self) -> Iterable[tuple[NodeT, NodeT]]: ... - - def successors(self, node: NodeT) -> frozenset[NodeT]: ... - - def predecessors(self, node: NodeT) -> frozenset[NodeT]: ... - - def in_degree(self, node: NodeT) -> int: ... - - def topological_sort(self) -> list[NodeT]: ... - - def ancestors(self, node: NodeT) -> frozenset[NodeT]: ... -``` - -**After line 348** (end of `topological_sort_deterministic`) — add `ancestors()` to `OrcaDAG`: -```python - def ancestors(self, node: NodeT) -> frozenset[NodeT]: - """Return all transitive predecessors of *node* via BFS. - - Args: - node: The node whose ancestors to find. - - Returns: - Frozen set of all nodes from which there is a directed path to - *node*. Empty if *node* is a source (no incoming edges). - - Raises: - KeyError: If *node* is not in the graph. - """ - _ = self._predecessors[node] # raise KeyError if absent - visited: set[NodeT] = set() - queue: list[NodeT] = list(self._predecessors[node]) - while queue: - n = queue.pop() - if n not in visited: - visited.add(n) - queue.extend(self._predecessors.get(n, set())) - return frozenset(visited) -``` - -- [ ] **Step 4: Update `test_networkx_backend.py` to use `GraphProtocol`** - -In `tests/test_pipeline/test_networkx_backend.py`, make these replacements: -- Line 14: `from orcapod.pipeline.dag import GraphBackend, OrcaDAG` → `from orcapod.pipeline.dag import GraphProtocol, OrcaDAG` -- Line 26: `assert isinstance(dag, GraphBackend)` → `assert isinstance(dag, GraphProtocol)` -- Line 30: `assert isinstance(backend, GraphBackend)` → `assert isinstance(backend, GraphProtocol)` -- Line 33: docstring `"""Verify both backends can be used interchangeably via GraphBackend."""` → `"""Verify both backends can be used interchangeably via GraphProtocol."""` -- Line 35: `def populate(g: GraphBackend[str]) -> None:` → `def populate(g: GraphProtocol[str]) -> None:` -- Lines 4–5: Update module docstring references from `GraphBackend` to `GraphProtocol` - -- [ ] **Step 5: Run all dag and networkx_backend tests** - -```bash -uv run pytest tests/test_pipeline/test_dag.py tests/test_pipeline/test_networkx_backend.py -v -``` - -Expected: all pass (new `ancestors()` tests now green; `GraphProtocol` rename tests still green). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/dag.py tests/test_pipeline/test_dag.py tests/test_pipeline/test_networkx_backend.py -git commit -m "refactor(pipeline): rename GraphBackend → GraphProtocol; add ancestors() to OrcaDAG" -``` - ---- - -## Task 4: Add `ancestors()` to `NetworkxBackend` - -**Files:** -- Modify: `src/orcapod/pipeline/networkx_backend.py` (docstring line ~46, after last method ~line 282) -- Modify: `tests/test_pipeline/test_networkx_backend.py` - -- [ ] **Step 1: Write failing test for `NetworkxBackend.ancestors()`** - -Add to `tests/test_pipeline/test_networkx_backend.py` (after existing tests): - -```python -class TestNetworkxBackendAncestors: - def test_ancestors_returns_transitive_predecessors(self): - from orcapod.pipeline.networkx_backend import NetworkxBackend - - backend = NetworkxBackend() - backend.add_edge("a", "b") - backend.add_edge("b", "c") - assert backend.ancestors("c") == frozenset({"a", "b"}) - assert backend.ancestors("b") == frozenset({"a"}) - assert backend.ancestors("a") == frozenset() -``` - -- [ ] **Step 2: Run to confirm failure** - -```bash -uv run pytest tests/test_pipeline/test_networkx_backend.py::TestNetworkxBackendAncestors -v -``` - -Expected: `AttributeError: 'NetworkxBackend' object has no attribute 'ancestors'` - -- [ ] **Step 3: Add `ancestors()` to `NetworkxBackend` and update docstring** - -In `src/orcapod/pipeline/networkx_backend.py`: - -**Docstring** (line ~46): change `satisfying the \`GraphBackend\` protocol` → `satisfying the \`GraphProtocol\` protocol` - -**After the last method** (after `topological_sort_deterministic`, ~line 282), add: - -```python - def ancestors(self, node: NodeT) -> frozenset[NodeT]: - """Return all transitive predecessors of *node*. - - Args: - node: The node whose ancestors to find. - - Returns: - Frozen set of all nodes from which there is a directed path to - *node*. - """ - import networkx as nx - - return frozenset(nx.ancestors(self._g, node)) -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/test_pipeline/test_networkx_backend.py -v -``` - -Expected: all pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/pipeline/networkx_backend.py tests/test_pipeline/test_networkx_backend.py -git commit -m "refactor(pipeline): add ancestors() to NetworkxBackend; update GraphProtocol reference" -``` - ---- - -## Task 5: Add `unbound_sources` property to `PipelineJob` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` (after `is_complete()` at line ~597) -- Modify: `tests/test_pipeline/test_pipeline_job.py` - -- [ ] **Step 1: Write failing tests for `unbound_sources`** - -Add a new test class to `tests/test_pipeline/test_pipeline_job.py` immediately after `TestPipelineJobCompleteness` (around line 294): - -```python -class TestUnboundSources: - def test_unbound_sources_returns_names_of_unbound_source_nodes(self, store): - """unbound_sources lists the name of each unbound SourceJobNode slot.""" - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(src_a, node_b) - - assert job.unbound_sources == ["spec_b"] - - def test_unbound_sources_empty_when_all_bound(self, store): - """unbound_sources is empty when all sources are bound.""" - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - assert job.unbound_sources == [] - - def test_unbound_sources_empty_before_compile(self): - """unbound_sources returns [] when job is not yet compiled.""" - job = PipelineJob() - assert job.unbound_sources == [] - - def test_unbound_sources_reflects_bind_call(self, store): - """After binding a source, it no longer appears in unbound_sources.""" - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(src_a, node_b) - - assert "spec_b" in job.unbound_sources - job.bind(sources={"spec_b": src_b}) - assert job.unbound_sources == [] -``` - -- [ ] **Step 2: Run to confirm failure** - -```bash -uv run pytest tests/test_pipeline/test_pipeline_job.py::TestUnboundSources -v -``` - -Expected: `AttributeError: 'PipelineJob' object has no attribute 'unbound_sources'` - -- [ ] **Step 3: Add `unbound_sources` property to `job.py`** - -In `src/orcapod/pipeline/job.py`, add the following property immediately after `is_complete()` (after line 597), inside the `# Completeness introspection` section: - -```python - @property - def unbound_sources(self) -> list[str]: - """Names of source slots not yet bound in this job. - - Computed live from ``_persistent_node_map`` — reflects the current - binding state at all times (before and after run). - - Returns: - List of unbound source slot names, in order of appearance in - the pipeline graph. Empty list if the job is not yet compiled. - """ - from orcapod.core.nodes.source_node import SourceJobNode - - if not self._compiled: - return [] - seen: set[str] = set() - result: list[str] = [] - for node in (self._persistent_node_map or {}).values(): - if ( - isinstance(node, SourceJobNode) - and node.bound_source is None - and node.name not in seen - ): - result.append(node.name) - seen.add(node.name) - return result -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/test_pipeline/test_pipeline_job.py::TestUnboundSources -v -``` - -Expected: all 4 pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/pipeline/job.py tests/test_pipeline/test_pipeline_job.py -git commit -m "feat(pipeline): add unbound_sources live-computed property to PipelineJob" -``` - ---- - -## Task 6: Remove old API, update `is_complete()`, `save()`, `load()`, `bind()` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` - -This task removes the old surface that `unbound_sources` now replaces, and updates related code. - -- [ ] **Step 1: Update `is_complete()` (line 591–597)** - -Replace: -```python - def is_complete(self) -> bool: - """Return ``True`` when all source nodes are bound and a store is set. - - Returns: - ``True`` if all SourceNode slots are bound and a store is set. - """ - return self._store is not None and len(self.unbound_source_nodes()) == 0 -``` - -With: -```python - def is_complete(self) -> bool: - """Return ``True`` when all source nodes are bound and a store is set. - - Returns: - ``True`` if all SourceNode slots are bound and a store is set. - """ - return self._store is not None and not self.unbound_sources -``` - -- [ ] **Step 2: Delete `unbound_source_nodes()` (lines 567–589)** - -Remove the entire `unbound_source_nodes()` method: -```python - def unbound_source_nodes(self) -> "list[Any]": - ... - return unbound -``` - -- [ ] **Step 3: Delete `unresolved_specs` property (lines 648–659)** - -Remove the entire `unresolved_specs` property block: -```python - # unresolved_specs property - # ... - @property - def unresolved_specs(self) -> list[str]: - ... - return list(self._unresolved_specs) -``` - -- [ ] **Step 4: Remove `_unresolved_specs` from `__init__` (line 84)** - -Remove this line from `__init__`: -```python - self._unresolved_specs: list[str] = [] -``` - -- [ ] **Step 5: Rename `spec_names` → `source_names` in `bind()` — first occurrence (lines 326–336)** - -Replace: -```python - spec_names = { - node.name - for node in (self._persistent_node_map or {}).values() - if isinstance(node, SourceNodeBase) - } - unknown = set(sources.keys()) - spec_names - if unknown: - raise ValueError( - f"bind() received source keys with no matching source slot " - f"(SourceJobNode): {sorted(unknown)}. " - f"Known slot names: {sorted(spec_names)}" - ) -``` - -With: -```python - source_names = { - node.name - for node in (self._persistent_node_map or {}).values() - if isinstance(node, SourceNodeBase) - } - unknown = set(sources.keys()) - source_names - if unknown: - raise ValueError( - f"bind() received source keys with no matching source slot " - f"(SourceJobNode): {sorted(unknown)}. " - f"Known slot names: {sorted(source_names)}" - ) -``` - -- [ ] **Step 6: Rename `spec_names` → `source_names` in `from_pipeline()` — second occurrence (lines 252–261)** - -Replace: -```python - spec_names = { - node.name - for node in pipeline._persistent_node_map.values() - if isinstance(node, SourceNodeBase) - } - unknown = set(bound_sources.keys()) - spec_names - if unknown: - raise ValueError( - f"from_pipeline() received source keys with no matching SourceNode: " - f"{sorted(unknown)}. Known names: {sorted(spec_names)}" - ) -``` - -With: -```python - source_names = { - node.name - for node in pipeline._persistent_node_map.values() - if isinstance(node, SourceNodeBase) - } - unknown = set(bound_sources.keys()) - source_names - if unknown: - raise ValueError( - f"from_pipeline() received source keys with no matching SourceNode: " - f"{sorted(unknown)}. Known names: {sorted(source_names)}" - ) -``` - -- [ ] **Step 7: Update `save()` JSON key (line 987–997)** - -Find the block that writes `"unresolved_specs"` to the run block and update the key: - -Replace: -```python - elif self._unresolved_specs: -``` -With: -```python - elif self.unbound_sources: -``` - -And replace: -```python - "unresolved_specs": list(self._unresolved_specs), -``` -With: -```python - "unbound_sources": list(self.unbound_sources), -``` - -- [ ] **Step 8: Update `load()` to remove `_unresolved_specs` restoration (line 1149)** - -Remove this line from `load()`: -```python - job._unresolved_specs = run_block.get("unresolved_specs", []) -``` - -(`unbound_sources` is live-computed from loaded `SourceJobNode` stubs — no restore needed.) - -- [ ] **Step 9: Run the completeness-related tests to confirm they still pass** - -```bash -uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobCompleteness tests/test_pipeline/test_pipeline_job.py::TestUnboundSources -v -``` - -Expected: all pass. - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/pipeline/job.py -git commit -m "refactor(pipeline): remove unbound_source_nodes/unresolved_specs; rename spec_names→source_names; update save/load" -``` - ---- - -## Task 7: Migrate all tests to new API - -**Files:** -- Modify: `tests/test_pipeline/test_pipeline_job.py` -- Modify: `tests/test_pipeline/test_serialization.py` - -- [ ] **Step 1: Update `TestPipelineJobCompleteness` to use `unbound_sources`** - -In `tests/test_pipeline/test_pipeline_job.py`, replace the two `unbound_source_nodes()` calls: - -**`test_unbound_specs_lists_unbound` (lines 246–258):** -```python - def test_unbound_specs_lists_unbound(self, store): - """unbound_sources lists names of unbound SourceJobNode slots.""" - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) - - job = PipelineJob(store=store) - with job: - Join()(src_a, node_b) - - assert job.unbound_sources == ["spec_b"] - - def test_unbound_specs_empty_when_all_bound(self, store): - src_a, src_b = _make_two_sources() - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b) - - assert job.unbound_sources == [] -``` - -- [ ] **Step 2: Update `test_run_executes_all_nodes` — `result.pipeline.compiled_nodes` → `result.compiled_nodes` (line 363)** - -Replace: -```python - node = result.pipeline.compiled_nodes["adder"] -``` -With: -```python - node = result.compiled_nodes["adder"] -``` - -- [ ] **Step 3: Update `test_run_produces_correct_values` (line 380)** - -Replace: -```python - table = result.pipeline.compiled_nodes["adder"].as_table() -``` -With: -```python - table = result.compiled_nodes["adder"].as_table() -``` - -- [ ] **Step 4: Update `test_run_partial_execution_skips_unbound_subgraph` (line 399)** - -Replace: -```python - assert "spec_b" in result.unresolved_specs -``` -With: -```python - assert "spec_b" in result.unbound_sources -``` - -- [ ] **Step 5: Rewrite `test_run_is_non_mutating` (lines 401–414)** - -Replace the entire method: -```python - def test_run_returns_self_and_mutates_in_place(self, store): - """run() returns self and sets _has_run / _run_id in place.""" - src_a, src_b = _make_two_sources() - - job = PipelineJob(store=store) - with job: - Join()(src_a, src_b, label="joiner") - - result = job.run() - assert result is job - assert job._has_run is True - assert job._run_id is not None -``` - -- [ ] **Step 6: Update `test_run_does_not_mutate_blueprint_nodes` docstring (lines 416–447)** - -Replace the docstring only (keep the test body — it still verifies blueprint._nodes is untouched): -```python - def test_run_does_not_mutate_blueprint_nodes(self, store): - """run() must not replace blueprint _nodes with execution nodes. - - The pipeline blueprint is a shared, reusable object. Running a job must - not replace blueprint template nodes with live exec nodes — that would - break subsequent jobs (and ``PipelineJob.from_pipeline()`` callers) that - share the same pipeline. - """ -``` - -- [ ] **Step 7: Update `test_run_twice_is_safe` (line 474)** - -Replace: -```python - table = result.pipeline.compiled_nodes["adder"].as_table() -``` -With: -```python - table = result.compiled_nodes["adder"].as_table() -``` - -- [ ] **Step 8: Update `test_load_after_partial_run_restores_unresolved_specs` (lines 663–679)** - -Replace the entire method: -```python - def test_load_after_partial_run_preserves_unbound_sources(self, store, tmp_path): - """Loaded job reports unbound sources for slots not yet bound.""" - src_a, src_b = _make_two_sources() - tag_b, data_b = src_b.output_schema() - node_b = SourceNode(name="unbound_b", tag_schema=tag_b, data_schema=data_b) - pf = PythonDataFunction(add_values, output_keys="total") - pod = FunctionPod(data_function=pf) - job = PipelineJob(store=store) - with job: - joined = Join()(src_a, node_b) - pod(joined, label="adder") - result = job.run() - assert "unbound_b" in result.unbound_sources - path = tmp_path / "partial.json" - result.save(str(path)) - loaded = PipelineJob.load(str(path), store=store) - assert "unbound_b" in loaded.unbound_sources -``` - -- [ ] **Step 9: Update `test_end_to_end_*` tests in `TestPipelineJobEndToEnd` — scan for `pipeline.compiled_nodes`** - -Search for any remaining `result.pipeline.compiled_nodes` or `.unresolved_specs` in the test file and replace: - -```bash -grep -n "pipeline\.compiled_nodes\|\.unresolved_specs\|unbound_source_nodes" tests/test_pipeline/test_pipeline_job.py -``` - -Fix any remaining occurrences using the same pattern: -- `result.pipeline.compiled_nodes["x"]` → `result.compiled_nodes["x"]` -- `result.unresolved_specs` → `result.unbound_sources` -- `job.unbound_source_nodes()` → `job.unbound_sources` - -- [ ] **Step 10: Update `test_serialization.py` (line 164)** - -Replace: -```python - assert completed.unresolved_specs == [] -``` -With: -```python - assert completed.unbound_sources == [] -``` - -- [ ] **Step 11: Run all migrated tests to see what passes and what still needs `run()` rewrite** - -```bash -uv run pytest tests/test_pipeline/test_pipeline_job.py tests/test_pipeline/test_serialization.py -v 2>&1 | tail -30 -``` - -Expected: completeness tests pass; `TestPipelineJobRun` tests may fail (they still call `run()` which still returns a new object — those will be fixed in Task 8). - -- [ ] **Step 12: Commit migrated tests** - -```bash -git add tests/test_pipeline/test_pipeline_job.py tests/test_pipeline/test_serialization.py -git commit -m "test(pipeline): migrate tests to unbound_sources; update run() return-value expectations" -``` - ---- - -## Task 8: Update orchestrators to accept `GraphProtocol` - -**Files:** -- Modify: `src/orcapod/pipeline/sync_orchestrator.py` -- Modify: `src/orcapod/pipeline/async_orchestrator.py` - -- [ ] **Step 1: Update `sync_orchestrator.py`** - -**Import section** — replace the `TYPE_CHECKING` block: -```python -if TYPE_CHECKING: - from orcapod.pipeline.dag import GraphProtocol - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol - from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -``` - -(Remove `import networkx as nx` from the `TYPE_CHECKING` block.) - -**`run()` signature** (line 48) — replace: -```python - def run( - self, - graph: "nx.DiGraph", -``` -With: -```python - def run( - self, - graph: "GraphProtocol[Any]", -``` - -**Inside `run()` body** (line 73–80) — remove the local `import networkx as nx` and replace: -```python - import networkx as nx - - run_id = run_id or str(uuid.uuid4()) - ... - topo_order = list(nx.topological_sort(graph)) -``` -With: -```python - run_id = run_id or str(uuid.uuid4()) - ... - topo_order = list(graph.topological_sort()) -``` - -**`run()` docstring** — update `graph` argument description: -```python - graph: A ``GraphProtocol`` DAG with GraphNode objects as vertices. -``` - -**`_gather_upstream()` signature** (line 122) — replace `graph: nx.DiGraph` with `graph: "GraphProtocol[Any]"`: -```python - def _gather_upstream( - node: Any, graph: "GraphProtocol[Any]", buffers: dict[Any, list[tuple[Any, Any]]] - ) -> list[tuple[Any, Any]]: -``` - -**`_gather_upstream_multi()` signature** (line 133) — same replacement: -```python - def _gather_upstream_multi( - node: Any, graph: "GraphProtocol[Any]", buffers: dict[Any, list[tuple[Any, Any]]] - ) -> list[tuple[list[tuple[Any, Any]], Any]]: -``` - -**`_gc_buffers()` signature** (line 217) — same replacement: -```python - def _gc_buffers( - current_node: Any, - graph: "GraphProtocol[Any]", - buffers: dict[Any, list[tuple[Any, Any]]], - processed: set[Any], - ) -> None: -``` - -- [ ] **Step 2: Update `async_orchestrator.py`** - -**Import section** — replace `import networkx as nx` in the `TYPE_CHECKING` block with: -```python -if TYPE_CHECKING: - from orcapod.pipeline.dag import GraphProtocol - ... -``` - -**`_run_async()` signature** (find `graph: nx.DiGraph` in the async method) — replace with `graph: "GraphProtocol[Any]"`. - -**Inside `_run_async()` body** (line 134–141) — remove `import networkx as nx` and replace: -```python - import networkx as nx - - run_id = run_id or str(uuid.uuid4()) - ... - topo_order = list(nx.topological_sort(graph)) -``` -With: -```python - run_id = run_id or str(uuid.uuid4()) - ... - topo_order = list(graph.topological_sort()) -``` - -**`graph.edges()` call** (line 147) — no change needed (already matches `GraphProtocol`): -```python - for upstream_node, downstream_node in graph.edges(): -``` - -- [ ] **Step 3: Run existing pipeline tests (they still pass since `run()` still passes nx.DiGraph)** - -```bash -uv run pytest tests/test_pipeline/ -v --ignore=tests/test_pipeline/test_pipeline_job.py -v 2>&1 | tail -20 -``` - -Expected: all pass (orchestrator tests not directly testing the graph type). - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/pipeline/sync_orchestrator.py src/orcapod/pipeline/async_orchestrator.py -git commit -m "refactor(pipeline): orchestrators accept GraphProtocol instead of nx.DiGraph" -``` - ---- - -## Task 9: Rewrite `run()`, remove `_build_execution_graph()` and `build_execution_graph()` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` - -This is the core task. After this, all migrated tests should pass. - -- [ ] **Step 1: Add `OrcaDAG` import to `job.py`** - -At the top of `src/orcapod/pipeline/job.py`, in the `TYPE_CHECKING` block (or directly if not already lazy), add: - -```python -from orcapod.pipeline.dag import OrcaDAG -``` - -Place this with the other non-conditional imports (it's a lightweight stdlib-only module — no lazy-loading needed). - -- [ ] **Step 2: Delete `build_execution_graph()` (lines 665–679)** - -Remove the entire public method: -```python - def build_execution_graph( - self, - ) -> "tuple[Any, list[str], Pipeline]": - ... - return self._build_execution_graph() -``` - -- [ ] **Step 3: Delete `_build_execution_graph()` (lines 681–839)** - -Remove the entire private method (lines 681–839). This is ~160 lines. - -- [ ] **Step 4: Rewrite `run()` (lines 845–916)** - -Replace the entire `run()` method with: - -```python - def run( - self, - observer: "ExecutionObserverProtocol | None" = None, - ) -> "PipelineJob": - """Execute the resolvable subgraph of this job in place. - - Nodes whose upstream includes an unbound source (and all their - dependents) are excluded from execution. Partial execution is a - first-class outcome — ``unbound_sources`` reports which source - slots were excluded. - - After a successful run, ``job.compiled_nodes`` returns the - execution-ready nodes with populated database caches. - - Args: - observer: Optional execution observer. - - Returns: - ``self`` — the same ``PipelineJob`` instance, with - ``_has_run`` and ``_run_id`` set. - - Raises: - ValueError: If no store is set. - RuntimeError: If no pipeline has been compiled. - """ - import hashlib - import uuid - - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.pipeline.observer import NoOpObserver - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - - if not self._compiled: - raise RuntimeError( - "PipelineJob has no compiled pipeline. " - "Record invocations inside a 'with job:' block first." - ) - if self._store is None: - raise ValueError( - "PipelineJob.run() requires a store. " - "Call job.bind(store=db) before run()." - ) - - # --- Build hash-keyed OrcaDAG for topological ordering + exclusion --- - hash_dag: OrcaDAG[str] = OrcaDAG() - for u, v in self._graph_edges: - hash_dag.add_edge(u, v) - for node_hash in self._persistent_node_map: - if node_hash not in hash_dag: - hash_dag.add_node(node_hash) - - # Walk topologically; exclude unbound sources and all their dependents. - excluded_hashes: set[str] = set() - for node_hash in hash_dag.topological_sort(): - if node_hash not in self._persistent_node_map: - continue - node = self._persistent_node_map[node_hash] - if isinstance(node, SourceJobNode) and node.bound_source is None: - excluded_hashes.add(node_hash) - elif any(p in excluded_hashes for p in hash_dag.predecessors(node_hash)): - excluded_hashes.add(node_hash) - - # --- Build execution OrcaDAG from existing node objects (no cloning) --- - exec_dag: OrcaDAG = OrcaDAG() - for node_hash, node in self._persistent_node_map.items(): - if node_hash not in excluded_hashes: - exec_dag.add_node(node) - for u_hash, v_hash in self._graph_edges: - if ( - u_hash not in excluded_hashes - and v_hash not in excluded_hashes - and u_hash in self._persistent_node_map - and v_hash in self._persistent_node_map - ): - exec_dag.add_edge( - self._persistent_node_map[u_hash], - self._persistent_node_map[v_hash], - ) - - # --- Execute --- - effective_observer = observer or NoOpObserver() - run_id = uuid.uuid4().hex[:16] - - # Snapshot hash: SHA-256 of sink-node content hashes for the run URI. - leaf_hashes = sorted( - node.content_hash().to_string() - for node in exec_dag.nodes() - if not exec_dag.successors(node) and hasattr(node, "content_hash") - ) - snapshot_hash = hashlib.sha256("\n".join(leaf_hashes).encode()).hexdigest()[:16] - pipeline_uri = "/".join(self._name) + "@" + snapshot_hash - - SyncPipelineOrchestrator().run( - exec_dag, - observer=effective_observer, - pipeline_uri=pipeline_uri, - ) - - # Flush databases. - pipeline_db = self._store.at(*self._name) - result_db = pipeline_db.at("_result") - pipeline_db.flush() - result_db.flush() - - # Mutate self in place. - self._has_run = True - self._run_id = run_id - - return self -``` - -- [ ] **Step 5: Run the full pipeline test suite** - -```bash -uv run pytest tests/test_pipeline/ -v 2>&1 | tail -40 -``` - -Expected: all pass. If any fail, read the error output carefully — the most likely issues are: -- A `result.pipeline.compiled_nodes` reference that was missed in Task 6 — fix it to `result.compiled_nodes` -- A `result.unresolved_specs` reference that was missed — fix it to `result.unbound_sources` -- A `result is not job` assertion — fix it to `result is job` - -- [ ] **Step 6: Run the broader test suite to check for regressions** - -```bash -uv run pytest tests/ -v 2>&1 | tail -50 -``` - -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/pipeline/job.py -git commit -m "refactor(pipeline): remove copy-on-run; inline exec graph in run(); drop _build_execution_graph()" -``` - ---- - -## Task 10: Final verification and push - -- [ ] **Step 1: Full clean test run** - -```bash -uv run pytest tests/ -v 2>&1 | tail -20 -``` - -Expected: all pass, no warnings about deprecated APIs. - -- [ ] **Step 2: Check DESIGN_ISSUES.md** - -```bash -grep -i "build_execution_graph\|unresolved_spec\|copy.on.run\|exec_pipeline" DESIGN_ISSUES.md -``` - -If any matching open issues exist, update their status to `resolved` and add a `Fix:` note referencing this PR. - -- [ ] **Step 3: Push branch** - -```bash -git push -u origin eywalker/eng-517-review-cleanup-of-pipelinejob-execution-graph-remove -``` - -- [ ] **Step 4: Create PR** - -```bash -gh pr create \ - --title "refactor(pipeline): remove copy-on-run pattern; adopt GraphProtocol; consolidate unbound_sources" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Removes the copy-on-run pattern from `PipelineJob.run()`: no more cloning of nodes, no `exec_pipeline`, no fresh `FunctionJobNode`/`OperatorJobNode` instances per run. -- `run()` now mutates `self` in place and returns `self`; databases were already wired at `bind(store=...)` time. -- `_build_execution_graph()` and `build_execution_graph()` deleted; execution graph construction inlined into `run()` as an `OrcaDAG[JobNode]`. -- Orchestrators updated to accept `GraphProtocol` instead of `nx.DiGraph`; three call-site substitutions cover all usage. -- `GraphBackend` renamed to `GraphProtocol` in `dag.py`/`networkx_backend.py`; `ancestors()` added to both. -- `unbound_source_nodes()` and `_unresolved_specs`/`unresolved_specs` removed; single live-computed `unbound_sources` property replaces both. -- `spec_names` local variable renamed to `source_names` in `bind()` and `from_pipeline()`. - -Closes ENG-517 - -## Test plan - -- [ ] `uv run pytest tests/test_pipeline/test_dag.py` — new `ancestors()` tests pass -- [ ] `uv run pytest tests/test_pipeline/test_networkx_backend.py` — `GraphProtocol` conformance + `ancestors()` pass -- [ ] `uv run pytest tests/test_pipeline/test_pipeline_job.py` — all run/completeness/load tests pass with new API -- [ ] `uv run pytest tests/test_pipeline/test_serialization.py` — `unbound_sources` key in saved JSON -- [ ] `uv run pytest tests/` — full suite green, no regressions - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - ---- - -## Self-Review Notes - -- **Spec coverage:** All 8 goals from the spec are addressed: `run()` mutates in place (Task 8), `_build_execution_graph()`/`build_execution_graph()` removed (Task 8), `exec_pipeline` gone (Task 8), orchestrators accept `GraphProtocol` (Task 7), `GraphProtocol` rename + `ancestors()` added (Tasks 2–3), `unbound_sources` replaces both old APIs (Tasks 4–5), `spec_names` → `source_names` (Task 5), `"unresolved_specs"` key → `"unbound_sources"` (Task 5), no regressions (Task 9). -- **No placeholders:** All steps include exact code. -- **Type consistency:** `GraphProtocol` used throughout Tasks 2–7; `OrcaDAG` introduced in Task 8; `unbound_sources` introduced in Task 4 and referenced consistently in Tasks 5–6. -- **`predecessors()` return type:** `OrcaDAG.predecessors()` returns `frozenset[NodeT]`. The `any(p in excluded_hashes ...)` call in Task 8 iterates a `frozenset` — correct. -- **`_graph_edges` availability:** Confirmed on `AbstractPipelineBase` (line 98 of `base.py`) — populated by `compile()`. Accessible as `self._graph_edges` in `run()`. -- **Backward compat:** No shims added per CLAUDE.md (`pre-v0.1.0 greenfield`). `load()` simply drops the `_unresolved_specs` restoration. diff --git a/superpowers/plans/2026-05-29-pipeline-dag-public-api.md b/superpowers/plans/2026-05-29-pipeline-dag-public-api.md deleted file mode 100644 index 3411d56b0..000000000 --- a/superpowers/plans/2026-05-29-pipeline-dag-public-api.md +++ /dev/null @@ -1,1035 +0,0 @@ -# Pipeline DAG Public API Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the private `_node_graph: nx.DiGraph` with `OrcaDAG[NodeT]`, expose it as a public `dag` property, make `AbstractPipelineBase` generic over its node type, and add a structural `PipelineProtocol[NodeT]`. - -**Architecture:** `AbstractPipelineBase[NodeT]` gains a `Generic[NodeT]` parent, `_node_graph` becomes `OrcaDAG[NodeT] | None`, and a `dag` property exposes it publicly. `PipelineProtocol[NodeT]` formalises the public surface. `Pipeline` binds `NodeT=GraphNode`; `PipelineJob` binds `NodeT=JobNode`. All four OrcaDAG-construction sites (compile, Pipeline.load, as_pipeline, PipelineJob.load) are updated to build `OrcaDAG` instead of `nx.DiGraph`. - -**Tech Stack:** Python generics (`typing.Generic`, `TypeVar`), `OrcaDAG` / `GraphProtocol` from `orcapod.pipeline.dag`, `pytest` via `uv run pytest`. - ---- - -## File Map - -| Action | File | What changes | -| ------ | ---- | ------------ | -| Modify | `src/orcapod/pipeline/base.py` | `Generic[NodeT]` parent, `_node_graph` type, `dag` property, compile() step 5 | -| Modify | `src/orcapod/protocols/pipeline_protocols.py` | Add `PipelineProtocol[NodeT]` | -| Modify | `src/orcapod/pipeline/graph.py` | `Pipeline(AbstractPipelineBase[GraphNode])`, load() OrcaDAG build, render_graph annotation | -| Modify | `src/orcapod/pipeline/job.py` | `PipelineJob(AbstractPipelineBase[JobNode])`, as_pipeline() OrcaDAG build, load() OrcaDAG build | -| Modify | `tests/test_pipeline/test_dag.py` | New tests: `dag` property, pre-compile error, `PipelineProtocol` conformance | -| Modify | `tests/test_pipeline/test_pipeline.py` | `_node_graph` → `dag` (4 sites) | -| Modify | `tests/test_pipeline/test_pipeline_job.py` | `_node_graph` → `dag` (1 site) | -| Modify | `tests/test_pipeline/test_serialization.py` | `_node_graph` → `dag` (2 sites) | -| Modify | `tests/test_pipeline/test_logging_observer_integration.py` | Helper `_get_function_node` | -| Modify | `tests/test_pipeline/test_graph_rendering.py` | Fixture return type + 2 sites | -| Modify | `tests/test_pipeline/test_status_observer_integration.py` | Helper + direct call | -| Modify | `tests/test_pipeline/test_orchestrator.py` | All `_node_graph` calls (~13 sites) | -| Modify | `tests/test_pipeline/test_orchestrator_executor_matrix.py` | 2 sites | -| Modify | `tests/test_pipeline/test_sync_orchestrator.py` | All `_node_graph` calls (~15 sites) | -| Modify | `tests/test_channels/test_pipeline_async_integration.py` | Delete `_build_exec_dag`, use `job.dag` | - ---- - -## Task 1: AbstractPipelineBase[NodeT] — generic base + dag property + compile() fix - -**Files:** -- Modify: `src/orcapod/pipeline/base.py:1-25` (imports), `base.py:46` (class decl), `base.py:88-91` (field types), `base.py:104-117` (properties), `base.py:357-366` (compile step 5) -- Test: `tests/test_pipeline/test_dag.py` - -- [ ] **Step 1: Write failing tests for `dag` property** - -Add a new test class to the bottom of `tests/test_pipeline/test_dag.py`: - -```python -# --------------------------------------------------------------------------- -# dag property on Pipeline / PipelineJob -# --------------------------------------------------------------------------- - - -class TestPipelineDagProperty: - """Pipeline.dag and PipelineJob.dag expose an OrcaDAG of node objects.""" - - def _make_simple_pipeline(self): - """Return a compiled single-function Pipeline.""" - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.core.function_pod import FunctionPod - from orcapod.core.data_function import PythonDataFunction - from orcapod.pipeline.graph import Pipeline - from orcapod.types import Schema - - src = ArrowTableSource( - pa.table({"id": pa.array(["a"], type=pa.large_string()), "v": pa.array([1], type=pa.int64())}), - tag_columns=["id"], - source_id="src", - ) - fn = PythonDataFunction( - func=lambda v: {"out": v}, - input_schema=Schema({"v": int}), - output_schema=Schema({"out": int}), - ) - pod = FunctionPod(fn) - pipeline = Pipeline(name="test_dag") - with pipeline: - pod(src) - return pipeline - - def test_dag_returns_orca_dag_after_compile(self): - """pipeline.dag returns an OrcaDAG instance after compilation.""" - pipeline = self._make_simple_pipeline() - assert isinstance(pipeline.dag, OrcaDAG) - - def test_dag_has_correct_node_count(self): - """dag contains source node + function node = 2 nodes.""" - pipeline = self._make_simple_pipeline() - assert len(list(pipeline.dag.nodes())) == 2 - - def test_dag_has_correct_edge(self): - """dag has exactly one edge (source → function).""" - pipeline = self._make_simple_pipeline() - assert len(list(pipeline.dag.edges())) == 1 - - def test_dag_topological_sort_works(self): - """dag.topological_sort() returns a 2-element list.""" - pipeline = self._make_simple_pipeline() - order = pipeline.dag.topological_sort() - assert len(order) == 2 - - def test_dag_predecessors_and_successors(self): - """Source node has no predecessors; function node is its successor.""" - from orcapod.core.nodes import SourceNode, FunctionNode - pipeline = self._make_simple_pipeline() - source = next(n for n in pipeline.dag.nodes() if isinstance(n, SourceNode)) - fn_node = next(n for n in pipeline.dag.nodes() if isinstance(n, FunctionNode)) - assert pipeline.dag.successors(source) == {fn_node} - assert pipeline.dag.predecessors(fn_node) == {source} - - def test_dag_raises_before_compile(self): - """Accessing dag before compile() raises RuntimeError.""" - from orcapod.pipeline.graph import Pipeline - pipeline = Pipeline(name="uncompiled", auto_compile=False) - with pytest.raises(RuntimeError, match="not been compiled"): - _ = pipeline.dag -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestPipelineDagProperty -v -``` - -Expected: FAIL with `AttributeError: 'Pipeline' object has no attribute 'dag'` - -- [ ] **Step 3: Add `Generic[NodeT]` to `AbstractPipelineBase`, add `dag` property, fix `compile()` step 5** - -In `src/orcapod/pipeline/base.py`, update the imports at the top: - -```python -"""AbstractPipelineBase — shared recording mechanism for Pipeline and PipelineJob.""" -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Generic, TypeVar - -from orcapod.core.tracker import AutoRegisteringContextBasedTracker -from orcapod.pipeline.dag import OrcaDAG -from orcapod.pipeline.pod_invocation import ( - FunctionInvocation, - OperatorInvocation, - PodInvocation, -) -from orcapod.protocols import core_protocols as cp -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import networkx as nx -else: - nx = LazyModule("networkx") - -logger = logging.getLogger(__name__) - -NodeT = TypeVar("NodeT") -``` - -Update the class declaration: - -```python -class AbstractPipelineBase(Generic[NodeT], AutoRegisteringContextBasedTracker, ABC): -``` - -Update the `__init__` field declarations (lines 88-91): - -```python - # --- Compiled state (populated / replaced by compile()) -------- - self._persistent_node_map: dict[str, NodeT] = {} # type: ignore[assignment] - self._nodes: dict[str, NodeT] = {} # type: ignore[assignment] - self._node_graph: OrcaDAG[NodeT] | None = None - self._compiled: bool = False -``` - -Add the `dag` property immediately after the existing `nodes` property (after line 117): - -```python - @property - def dag(self) -> OrcaDAG[NodeT]: - """Node-object DAG for this pipeline. - - Returns an ``OrcaDAG`` whose vertices are the compiled node objects - (``GraphNode`` for ``Pipeline``, ``JobNode`` for ``PipelineJob``) and - whose edges follow the data-flow topology. - - Raises: - RuntimeError: If the pipeline has not been compiled yet. - """ - if self._node_graph is None: - raise RuntimeError( - "Pipeline has not been compiled. " - "Use 'with pipeline:' or call compile() first." - ) - return self._node_graph -``` - -Replace compile() step 5 (lines 357-366) — replace the `_nx.DiGraph()` block: - -```python - # 5. Build node_graph (OrcaDAG with node objects as vertices). - node_dag: OrcaDAG[Any] = OrcaDAG() - for up_hash, down_hash in self._hash_graph.edges(): - up_node = node_map.get(up_hash) - down_node = node_map.get(down_hash) - if up_node is not None and down_node is not None: - node_dag.add_edge(up_node, down_node) - for node in node_map.values(): - if node not in node_dag: - node_dag.add_node(node) - self._node_graph = node_dag # type: ignore[assignment] -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestPipelineDagProperty -v -``` - -Expected: PASS (all 6 tests green) - -- [ ] **Step 5: Run full test suite to check regressions** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: Some failures in test files that call `nx.topological_sort(pipeline._node_graph)` — those will be fixed in Task 6. All other tests should still pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/base.py tests/test_pipeline/test_dag.py -git commit -m "feat(pipeline): make AbstractPipelineBase generic and add dag property - -NodeT TypeVar, OrcaDAG[NodeT] _node_graph, dag property with RuntimeError guard, -compile() step 5 now builds OrcaDAG instead of nx.DiGraph. - -Co-Authored-By: Claude Sonnet 4.6 " -``` - ---- - -## Task 2: PipelineProtocol[NodeT] - -**Files:** -- Modify: `src/orcapod/protocols/pipeline_protocols.py` -- Modify: `tests/test_pipeline/test_dag.py` - -- [ ] **Step 1: Write failing test for structural conformance** - -Add to `tests/test_pipeline/test_dag.py`: - -```python -# --------------------------------------------------------------------------- -# PipelineProtocol structural conformance -# --------------------------------------------------------------------------- - - -class TestPipelineProtocolConformance: - """Pipeline and PipelineJob satisfy PipelineProtocol structurally.""" - - def test_pipeline_satisfies_protocol(self): - """Pipeline is runtime-checkable as PipelineProtocol.""" - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.pipeline.graph import Pipeline - from orcapod.protocols.pipeline_protocols import PipelineProtocol - - src = ArrowTableSource( - pa.table({"id": pa.array(["a"], type=pa.large_string()), "v": pa.array([1], type=pa.int64())}), - tag_columns=["id"], - source_id="src", - ) - pipeline = Pipeline(name="proto_test") - with pipeline: - pass # no ops, just source wrapping - - assert isinstance(pipeline, PipelineProtocol) - - def test_pipeline_job_satisfies_protocol(self): - """PipelineJob is runtime-checkable as PipelineProtocol.""" - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.databases.in_memory_arrow_database import InMemoryArrowDatabase - from orcapod.pipeline.job import PipelineJob - from orcapod.protocols.pipeline_protocols import PipelineProtocol - - src = ArrowTableSource( - pa.table({"id": pa.array(["a"], type=pa.large_string()), "v": pa.array([1], type=pa.int64())}), - tag_columns=["id"], - source_id="src", - ) - job = PipelineJob(store=InMemoryArrowDatabase()) - with job: - pass - - assert isinstance(job, PipelineProtocol) - - def test_protocol_dag_returns_graph_protocol(self): - """dag attribute declared on PipelineProtocol returns GraphProtocol.""" - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.pipeline.graph import Pipeline - from orcapod.pipeline.dag import GraphProtocol - - src = ArrowTableSource( - pa.table({"id": pa.array(["a"], type=pa.large_string()), "v": pa.array([1], type=pa.int64())}), - tag_columns=["id"], - source_id="src", - ) - pipeline = Pipeline(name="proto_dag_test") - with pipeline: - pass - - assert isinstance(pipeline.dag, GraphProtocol) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestPipelineProtocolConformance -v -``` - -Expected: FAIL with `ImportError: cannot import name 'PipelineProtocol'` (or `AttributeError`) - -- [ ] **Step 3: Add `PipelineProtocol[NodeT]` to `pipeline_protocols.py`** - -Replace the full contents of `src/orcapod/protocols/pipeline_protocols.py`: - -```python -# Protocols for pipeline and nodes -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol, TypeVar, runtime_checkable - -from orcapod.protocols import core_protocols as cp - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.pipeline.dag import GraphProtocol - -NodeT = TypeVar("NodeT") - - -@runtime_checkable -class PipelineProtocol(Protocol[NodeT]): - """Structural protocol for ``Pipeline`` and ``PipelineJob``. - - Both ``Pipeline`` (``NodeT=GraphNode``) and ``PipelineJob`` - (``NodeT=JobNode``) satisfy this protocol. Callers that only need - DAG introspection can accept ``PipelineProtocol[Any]`` rather than - importing the concrete classes. - - Note: - The ``dag`` return type is ``GraphProtocol[NodeT]`` here (the abstract - protocol). Callers using the concrete classes receive the more - specific ``OrcaDAG[NodeT]`` type. - """ - - @property - def name(self) -> tuple[str, ...]: - """Pipeline name tuple.""" - ... - - @property - def nodes(self) -> dict[str, NodeT]: - """Copy of the compiled label → node mapping.""" - ... - - @property - def dag(self) -> "GraphProtocol[NodeT]": - """Node-object DAG for topology traversal and introspection.""" - ... - - -class NodeProtocol(cp.Source, Protocol): - # def record_pipeline_outputs(self): - # pass - ... - - -@runtime_checkable -class PodNodeProtocol(cp.CachedPod, Protocol): - def get_all_records( - self, include_system_columns: bool = False - ) -> "pa.Table | None": - """ - Retrieve all tag and data processed by this PodProtocol. - - This method returns a table containing all data processed by the PodProtocol, - including metadata and system columns if requested. It is useful for: - - Debugging and analysis - - Auditing and data lineage tracking - - Performance monitoring - - Args: - include_system_columns: Whether to include system columns in the output - - Returns: - pa.Table | None: A table containing all processed records, or None if no records are available - """ - ... - - def flush(self): - """ - Flush any in-memory data to persistent storage. - - This method ensures that all buffered data is written to the underlying - storage system, making it durable and consistent. It is useful for: - - Ensuring data integrity before shutdown or restart - - Committing changes after a batch of operations - - Reducing memory usage by clearing buffers - - """ - ... - - def add_pipeline_record( - self, - tag: cp.TagProtocol, - input_data: cp.DataProtocol, - data_record_id: str, - retrieved: bool | None = None, - skip_cache_lookup: bool = False, - ) -> None: ... -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestPipelineProtocolConformance -v -``` - -Expected: PASS (all 3 tests green) - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/protocols/pipeline_protocols.py tests/test_pipeline/test_dag.py -git commit -m "feat(protocols): add PipelineProtocol[NodeT] - -Generic structural protocol covering name, nodes, and dag. -Runtime-checkable so isinstance checks work. - -Co-Authored-By: Claude Sonnet 4.6 " -``` - ---- - -## Task 3: Pipeline(AbstractPipelineBase[GraphNode]) + Pipeline.load() + render_graph annotation - -**Files:** -- Modify: `src/orcapod/pipeline/graph.py:34` (class decl), `graph.py:369-377` (load OrcaDAG), `graph.py:726-741` (render_graph annotation) - -- [ ] **Step 1: Update `Pipeline` class declaration** - -In `src/orcapod/pipeline/graph.py`, update the import at the top to include `GraphNode`: - -```python -from orcapod.core.nodes import ( - FunctionNode, - GraphNode, - OperatorNode, - SourceNode, -) -``` - -(This import already exists — `GraphNode` is already imported. No change needed.) - -Change the class declaration at line 34: - -```python -class Pipeline(AbstractPipelineBase[GraphNode]): -``` - -- [ ] **Step 2: Update `Pipeline.load()` to build OrcaDAG** - -In `src/orcapod/pipeline/graph.py`, add `OrcaDAG` to the imports at the top of the file: - -```python -from orcapod.pipeline.dag import OrcaDAG -``` - -Replace the `_node_graph` construction block in `Pipeline.load()` (lines 369-377): - -```python - node_dag: OrcaDAG[GraphNode] = OrcaDAG() - for up_hash, down_hash in edges: - up_node = reconstructed.get(up_hash) - down_node = reconstructed.get(down_hash) - if up_node is not None and down_node is not None: - node_dag.add_edge(up_node, down_node) - for node in reconstructed.values(): - if node not in node_dag: - node_dag.add_node(node) - pipeline._node_graph = node_dag -``` - -- [ ] **Step 3: Update `render_graph` type annotations** - -In `src/orcapod/pipeline/graph.py`, update the `GraphRenderer.generate_dot` signature (line 601-606), `GraphRenderer.render_graph` (line 645-648), the module-level `render_graph` (line 726-729), and `render_graph_dark_theme` (line 744-746) to accept `GraphProtocol[Any]` instead of `nx.DiGraph`: - -Add to the `TYPE_CHECKING` block at the top of `graph.py`: - -```python -if TYPE_CHECKING: - import networkx as nx - from orcapod.pipeline.execution_context import ExecutionContext -``` - -becomes: - -```python -if TYPE_CHECKING: - import networkx as nx - from orcapod.pipeline.dag import GraphProtocol - from orcapod.pipeline.execution_context import ExecutionContext -``` - -Then update the four signatures: - -```python - def generate_dot( - self, - graph: "GraphProtocol[GraphNode]", - ... - ) -> str: -``` - -```python - def render_graph( - self, - graph: "GraphProtocol[GraphNode]", - ... - ) -> str | None: -``` - -```python -def render_graph( - graph: "GraphProtocol[GraphNode]", - ... -) -> str | None: -``` - -```python -def render_graph_dark_theme( - graph: "GraphProtocol[GraphNode]", - ... -) -> str | None: -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/test_pipeline/test_serialization.py -v -k "save_load" -``` - -Expected: The `test_save_load_preserves_graph_shape` test now uses `loaded._node_graph` directly — it still fails because the test calls `_node_graph` (will be fixed in Task 6). Other serialization tests should pass. - -```bash -uv run pytest tests/test_pipeline/test_dag.py -v -``` - -Expected: PASS (all dag tests still green) - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/pipeline/graph.py -git commit -m "feat(pipeline): Pipeline[GraphNode] + load() OrcaDAG + render_graph annotation - -Pipeline class now explicitly parameterised as AbstractPipelineBase[GraphNode]. -Pipeline.load() builds OrcaDAG[GraphNode] instead of nx.DiGraph. -render_graph/generate_dot accept GraphProtocol[GraphNode]. - -Co-Authored-By: Claude Sonnet 4.6 " -``` - ---- - -## Task 4: PipelineJob(AbstractPipelineBase[JobNode]) + as_pipeline() OrcaDAG fix - -**Files:** -- Modify: `src/orcapod/pipeline/job.py:28` (class decl), `job.py:530-540` (as_pipeline extracted pipeline._node_graph) - -- [ ] **Step 1: Update `PipelineJob` class declaration** - -In `src/orcapod/pipeline/job.py`, add `JobNode` to the top-level imports: - -```python -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.nodes.operator_node import OperatorJobNode -from orcapod.core.nodes.source_node import SourceJobNode -``` - -becomes: - -```python -from orcapod.core.nodes import JobNode -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.nodes.operator_node import OperatorJobNode -from orcapod.core.nodes.source_node import SourceJobNode -``` - -Change the class declaration at line 28: - -```python -class PipelineJob(AbstractPipelineBase[JobNode]): -``` - -- [ ] **Step 2: Fix the extracted `pipeline._node_graph` build in `as_pipeline()`** - -In `src/orcapod/pipeline/job.py`, find the block starting with `# Build _node_graph (DiGraph with node objects as vertices).` around line 530. Replace it: - -```python - # Build _node_graph (OrcaDAG with node objects as vertices). - # Mirrors AbstractPipelineBase.compile() step 5. - # node_map here contains GraphNode objects (blueprint nodes). - from orcapod.pipeline.dag import OrcaDAG as _OrcaDAG - bp_dag: _OrcaDAG = _OrcaDAG() - for up_h, down_h in self._hash_graph.edges(): - up_node = node_map.get(up_h) - down_node = node_map.get(down_h) - if up_node is not None and down_node is not None: - bp_dag.add_edge(up_node, down_node) - for node in node_map.values(): - if node not in bp_dag: - bp_dag.add_node(node) - pipeline._node_graph = bp_dag -``` - -Note: `OrcaDAG` is already imported at the top of `job.py` (`from orcapod.pipeline.dag import OrcaDAG`). Use it directly without the local alias: - -```python - # Build _node_graph (OrcaDAG with node objects as vertices). - # Mirrors AbstractPipelineBase.compile() step 5. - # node_map here contains GraphNode objects (blueprint nodes). - bp_dag: OrcaDAG = OrcaDAG() - for up_h, down_h in self._hash_graph.edges(): - up_node = node_map.get(up_h) - down_node = node_map.get(down_h) - if up_node is not None and down_node is not None: - bp_dag.add_edge(up_node, down_node) - for node in node_map.values(): - if node not in bp_dag: - bp_dag.add_node(node) - pipeline._node_graph = bp_dag -``` - -Also remove the now-unused `_nx` local import in `as_pipeline()`. Find the line: -```python - import networkx as _nx -``` -and remove it (it was used only for `_nx.DiGraph()`). - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/test_pipeline/test_dag.py tests/test_pipeline/test_pipeline.py -v -q 2>&1 | tail -20 -``` - -Expected: `TestPipelineDagProperty` and `TestPipelineProtocolConformance` still pass. `test_pipeline.py` failures at `_node_graph` sites are expected (fixed in Task 6). - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/pipeline/job.py -git commit -m "feat(pipeline): PipelineJob[JobNode] + as_pipeline() OrcaDAG fix - -PipelineJob explicitly parameterised as AbstractPipelineBase[JobNode]. -as_pipeline() now builds OrcaDAG for the extracted Pipeline blueprint. - -Co-Authored-By: Claude Sonnet 4.6 " -``` - ---- - -## Task 5: PipelineJob.load() — fix job._node_graph to OrcaDAG[JobNode] - -**Files:** -- Modify: `src/orcapod/pipeline/job.py:1080-1083` -- Test: `tests/test_pipeline/test_dag.py` - -- [ ] **Step 1: Write failing test for `job.dag` on a loaded job** - -Add to `tests/test_pipeline/test_dag.py`: - -```python -class TestPipelineJobDagOnLoadedJob: - """job.dag returns OrcaDAG[JobNode] even for loaded (not compiled) jobs.""" - - def test_loaded_job_dag_returns_job_nodes(self, tmp_path): - """PipelineJob.load() must produce OrcaDAG with JobNode objects.""" - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.core.function_pod import FunctionPod - from orcapod.core.data_function import PythonDataFunction - from orcapod.core.nodes import JobNode - from orcapod.databases.in_memory_arrow_database import InMemoryArrowDatabase - from orcapod.pipeline.graph import Pipeline - from orcapod.pipeline.job import PipelineJob - from orcapod.types import Schema - - src = ArrowTableSource( - pa.table({"id": pa.array(["a"], type=pa.large_string()), "v": pa.array([1], type=pa.int64())}), - tag_columns=["id"], - source_id="src", - ) - fn = PythonDataFunction( - func=lambda v: {"out": v}, - input_schema=Schema({"v": int}), - output_schema=Schema({"out": int}), - ) - pod = FunctionPod(fn) - - # Build, run, and save a pipeline - pipeline = Pipeline(name="load_dag_test") - with pipeline: - pod(src) - job = PipelineJob.from_pipeline( - pipeline, sources={"src": src}, store=InMemoryArrowDatabase() - ) - job.run() - save_path = tmp_path / "job.json" - job.save(str(save_path)) - - # Load and check dag - loaded_job = PipelineJob.load(str(save_path)) - assert isinstance(loaded_job.dag, OrcaDAG) - # All nodes in the loaded job's dag must be JobNode instances - for node in loaded_job.dag.nodes(): - assert isinstance(node, JobNode), ( - f"Expected JobNode, got {type(node).__name__}" - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestPipelineJobDagOnLoadedJob -v -``` - -Expected: FAIL — the loaded job's dag currently contains `GraphNode` (blueprint) objects instead of `JobNode`. - -- [ ] **Step 3: Fix `PipelineJob.load()` to build `OrcaDAG[JobNode]`** - -In `src/orcapod/pipeline/job.py`, find the block around line 1080: - -```python - # Copy pipeline._node_graph for graph-introspection use (renderers, etc.). - # Nodes here are SourceNode / FunctionNode / OperatorNode from the blueprint; - # is_runnable() does not use this graph and is unaffected. - job._node_graph = pipeline._node_graph -``` - -Replace it with: - -```python - # Build job._node_graph as OrcaDAG[JobNode] from the job's own persistent - # node map. This makes job.dag consistent with the compiled-job path - # (where compile() builds _node_graph from job nodes). - # Callers needing blueprint nodes for rendering use job.pipeline.dag. - job_dag: OrcaDAG = OrcaDAG() - for node in job._persistent_node_map.values(): - job_dag.add_node(node) - for u_hash, v_hash in job._graph_edges: - u_node = job._persistent_node_map.get(u_hash) - v_node = job._persistent_node_map.get(v_hash) - if u_node is not None and v_node is not None: - job_dag.add_edge(u_node, v_node) - job._node_graph = job_dag -``` - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/test_pipeline/test_dag.py::TestPipelineJobDagOnLoadedJob -v -``` - -Expected: PASS - -- [ ] **Step 5: Run full dag test suite** - -```bash -uv run pytest tests/test_pipeline/test_dag.py -v -``` - -Expected: All tests pass (including `TestPipelineDagProperty`, `TestPipelineProtocolConformance`, `TestPipelineJobDagOnLoadedJob`). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/job.py tests/test_pipeline/test_dag.py -git commit -m "fix(pipeline): PipelineJob.load() builds OrcaDAG[JobNode] for job.dag - -Previously copied blueprint (GraphNode) objects from pipeline._node_graph. -Now builds OrcaDAG from _persistent_node_map so job.dag is consistent -with the compiled-job path. Callers needing blueprint nodes use job.pipeline.dag. - -Co-Authored-By: Claude Sonnet 4.6 " -``` - ---- - -## Task 6: Migrate all test call sites from `_node_graph` to `dag` - -**Files:** -- Modify: `tests/test_pipeline/test_pipeline.py` -- Modify: `tests/test_pipeline/test_pipeline_job.py` -- Modify: `tests/test_pipeline/test_serialization.py` -- Modify: `tests/test_pipeline/test_logging_observer_integration.py` -- Modify: `tests/test_pipeline/test_graph_rendering.py` -- Modify: `tests/test_pipeline/test_status_observer_integration.py` -- Modify: `tests/test_pipeline/test_orchestrator.py` -- Modify: `tests/test_pipeline/test_orchestrator_executor_matrix.py` -- Modify: `tests/test_pipeline/test_sync_orchestrator.py` -- Modify: `tests/test_channels/test_pipeline_async_integration.py` - -- [ ] **Step 1: Migrate `test_pipeline.py` (4 sites)** - -Replace all 4 occurrences of `pipeline._node_graph.nodes()` with `pipeline.dag.nodes()`: - -```python -# Line 102: was -source_nodes = [ - n for n in pipeline._node_graph.nodes() if isinstance(n, SourceNode) -] -# becomes -source_nodes = [ - n for n in pipeline.dag.nodes() if isinstance(n, SourceNode) -] - -# Line 115: same pattern -source_nodes = [n for n in pipeline.dag.nodes() if isinstance(n, SourceNode)] - -# Line 156: same pattern -source_nodes = [n for n in pipeline.dag.nodes() if isinstance(n, SourceNode)] - -# Line 678: was job.pipeline._node_graph -source_nodes = [ - n for n in job.pipeline.dag.nodes() if isinstance(n, SourceNode) -] -``` - -- [ ] **Step 2: Migrate `test_pipeline_job.py` (1 site)** - -```python -# Line 73: was job.pipeline._node_graph -source_nodes = [ - n for n in job.pipeline.dag.nodes() if isinstance(n, SourceNode) -] -``` - -- [ ] **Step 3: Migrate `test_serialization.py` (2 sites)** - -```python -# Line 83: was -assert len(list(loaded._node_graph.edges())) == len(list(pipeline._node_graph.edges())) -# becomes -assert len(list(loaded.dag.edges())) == len(list(pipeline.dag.edges())) -``` - -- [ ] **Step 4: Migrate `test_logging_observer_integration.py`** - -Replace the `_get_function_node` helper (lines 39-46): - -```python -def _get_function_node(pipeline: Pipeline): - """Return the first function node from the pipeline graph.""" - for node in pipeline.dag.topological_sort(): - if node.node_type == "function": - return node - raise RuntimeError("No function node found") -``` - -Remove the `import networkx as nx` inside the function (it's no longer needed). - -- [ ] **Step 5: Migrate `test_graph_rendering.py` (fixture + 2 sites)** - -Update the `node_graph` fixture (lines 80-83): - -```python -@pytest.fixture -def node_graph(compiled_pipeline: Pipeline) -> OrcaDAG: - assert compiled_pipeline._compiled - return compiled_pipeline.dag -``` - -Add `from orcapod.pipeline.dag import OrcaDAG` to the imports at the top of the test file, and remove `import networkx as nx` if it's only used for this fixture. - -Update lines 319-320: - -```python -# was: assert compiled_pipeline._node_graph is not None -assert compiled_pipeline._compiled -# was: first_node = next(iter(compiled_pipeline._node_graph.nodes())) -first_node = next(iter(compiled_pipeline.dag.nodes())) -``` - -- [ ] **Step 6: Migrate `test_status_observer_integration.py`** - -Replace the `_get_function_node` helper (same pattern as logging): - -```python -def _get_function_node(pipeline: Pipeline): - """Return the first function node from the pipeline graph.""" - for node in pipeline.dag.topological_sort(): - if node.node_type == "function": - return node - raise RuntimeError("No function node found") -``` - -Replace line 454: -```python -# was -orch.run(pipeline._node_graph, observer=obs, run_id="my-custom-run-id") -# becomes -orch.run(pipeline.dag, observer=obs, run_id="my-custom-run-id") -``` - -- [ ] **Step 7: Migrate `test_orchestrator.py` (~13 sites)** - -Replace every `pipeline._node_graph` with `pipeline.dag`: - -```python -# All occurrences of the pattern: -AsyncPipelineOrchestrator().run(pipeline._node_graph) -# become: -AsyncPipelineOrchestrator().run(pipeline.dag) - -# And: -await orchestrator.run_async(pipeline._node_graph) -# becomes: -await orchestrator.run_async(pipeline.dag) - -# And: -orch.run(pipeline._node_graph, ...) -# becomes: -orch.run(pipeline.dag, ...) - -# And: -result = orch.run(pipeline._node_graph, materialize_results=True) -# becomes: -result = orch.run(pipeline.dag, materialize_results=True) -``` - -Run a quick verify after this file: - -```bash -uv run pytest tests/test_pipeline/test_orchestrator.py -v -q 2>&1 | tail -10 -``` - -- [ ] **Step 8: Migrate `test_orchestrator_executor_matrix.py` (2 sites)** - -```python -# Line 177: -result = orch.run(pipeline.dag) -# Line 511: -await AsyncPipelineOrchestrator().run_async(pipeline.dag) -``` - -- [ ] **Step 9: Migrate `test_sync_orchestrator.py` (~15 sites)** - -Replace every `pipeline._node_graph` and `async_pipeline._node_graph` with `.dag`: - -```python -# All occurrences: -orch.run(pipeline._node_graph) → orch.run(pipeline.dag) -orch.run(pipeline._node_graph, ...) → orch.run(pipeline.dag, ...) -AsyncPipelineOrchestrator().run(async_pipeline._node_graph) - → AsyncPipelineOrchestrator().run(async_pipeline.dag) -``` - -- [ ] **Step 10: Migrate `test_pipeline_async_integration.py` — delete `_build_exec_dag`** - -Delete the `_build_exec_dag` helper function (lines 103-120). - -Replace every call `exec_dag = _build_exec_dag(job)` with `exec_dag = job.dag`: - -```python -# was: -exec_dag = _build_exec_dag(job) -AsyncPipelineOrchestrator().run(exec_dag) -# becomes: -AsyncPipelineOrchestrator().run(job.dag) - -# was: -exec_dag = _build_exec_dag(async_job) -AsyncPipelineOrchestrator().run(exec_dag) -# becomes: -AsyncPipelineOrchestrator().run(async_job.dag) - -# And for run_async: -exec_dag = _build_exec_dag(job) -await orchestrator.run_async(exec_dag) -# becomes: -await orchestrator.run_async(job.dag) -``` - -- [ ] **Step 11: Run the full test suite** - -```bash -uv run pytest tests/ -q 2>&1 | tail -20 -``` - -Expected: All tests pass. Zero failures. - -- [ ] **Step 12: Commit** - -```bash -git add \ - tests/test_pipeline/test_pipeline.py \ - tests/test_pipeline/test_pipeline_job.py \ - tests/test_pipeline/test_serialization.py \ - tests/test_pipeline/test_logging_observer_integration.py \ - tests/test_pipeline/test_graph_rendering.py \ - tests/test_pipeline/test_status_observer_integration.py \ - tests/test_pipeline/test_orchestrator.py \ - tests/test_pipeline/test_orchestrator_executor_matrix.py \ - tests/test_pipeline/test_sync_orchestrator.py \ - tests/test_channels/test_pipeline_async_integration.py -git commit -m "refactor(tests): migrate _node_graph → dag across all test files - -All test call sites updated to use the public dag property. -_build_exec_dag helper in async integration tests deleted (job.dag replaces it). -import networkx as nx removed where no longer needed. - -Co-Authored-By: Claude Sonnet 4.6 " -``` - ---- - -## Self-Review Checklist - -- [x] **Spec coverage:** `PipelineProtocol[NodeT]` ✓ Task 2. `AbstractPipelineBase[NodeT]` ✓ Task 1. `_node_graph` → `OrcaDAG` in all four sites ✓ Tasks 1, 3, 4, 5. `dag` property with guard ✓ Task 1. Call-site migration ✓ Task 6. New tests ✓ Tasks 1, 2, 5. -- [x] **Placeholder scan:** No TBDs or "add appropriate handling" patterns found. -- [x] **Type consistency:** `OrcaDAG` (not `OrcaDAG[Any]`) used consistently in non-generic contexts. `dag` property returns `OrcaDAG[NodeT]` in base, `GraphProtocol[NodeT]` in protocol. diff --git a/superpowers/plans/2026-06-02-dataclass-arrow-struct-encoding.md b/superpowers/plans/2026-06-02-dataclass-arrow-struct-encoding.md deleted file mode 100644 index 308c8b45d..000000000 --- a/superpowers/plans/2026-06-02-dataclass-arrow-struct-encoding.md +++ /dev/null @@ -1,1158 +0,0 @@ -# Dataclass Arrow Struct Encoding Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `@dataclass` serialization to Orcapod as Arrow structs with a `__type` sentinel field and three-tier (import → registry → synthesize) deserialization, wired automatically into the existing `UniversalTypeConverter` pipeline. - -**Architecture:** A self-contained `dataclass_encoding.py` module handles all logic (encoding, decoding, registry, cache). `UniversalTypeConverter` gains thin shims in four methods that delegate to it when a dataclass type or `__type`-bearing struct is detected. No changes are needed outside these two source files plus the public `__init__.py`. - -**Tech Stack:** Python `dataclasses`, `importlib`, `re`, `typing`, `pyarrow`, `unittest.mock` (tests only) - -**Branch:** `eywalker/eng-555-add-dataclass-support-to-orcapod-via-arrow-struct-encoding` - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/semantic_types/dataclass_encoding.py` | **Create** | All encode/decode logic, registry, three-tier decoder | -| `src/orcapod/semantic_types/universal_converter.py` | **Modify** | 4 shim branches + `__init__` + `clear_cache` | -| `src/orcapod/__init__.py` | **Modify** | Expose `register_dataclass` in public API | -| `tests/test_semantic_types/test_dataclass_encoding.py` | **Create** | All unit + integration tests | - ---- - -## Prerequisite: Verify branch - -Before any task, confirm you are on the correct branch: - -```bash -git branch --show-current -# Expected: eywalker/eng-555-add-dataclass-support-to-orcapod-via-arrow-struct-encoding -``` - ---- - -### Task 1: Module skeleton — constants and registry - -**Files:** -- Create: `src/orcapod/semantic_types/dataclass_encoding.py` -- Create: `tests/test_semantic_types/test_dataclass_encoding.py` - -- [ ] **Step 1: Write the failing tests** - -```python -# tests/test_semantic_types/test_dataclass_encoding.py -from __future__ import annotations - -import dataclasses -import typing - -import pytest - -from orcapod.semantic_types.dataclass_encoding import ( - DATACLASS_TYPE_FIELD, - DATACLASS_TYPE_PREFIX, - _DATACLASS_REGISTRY, - register_dataclass, -) - - -@dataclasses.dataclass -class _Simple: - a: int - b: str - - -def test_constants(): - assert DATACLASS_TYPE_FIELD == "__type" - assert DATACLASS_TYPE_PREFIX == "dataclass:" - - -def test_register_explicit(): - register_dataclass(_Simple) - key = f"{_Simple.__module__}.{_Simple.__qualname__}" - assert _DATACLASS_REGISTRY[key] is _Simple - - -def test_register_returns_class(): - result = register_dataclass(_Simple) - assert result is _Simple - - -def test_register_as_decorator(): - @register_dataclass - @dataclasses.dataclass - class _Decorated: - x: float - - key = f"{_Decorated.__module__}.{_Decorated.__qualname__}" - assert _DATACLASS_REGISTRY[key] is _Decorated -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -v -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.semantic_types.dataclass_encoding'` - -- [ ] **Step 3: Create `dataclass_encoding.py` with constants and registry** - -```python -# src/orcapod/semantic_types/dataclass_encoding.py -""" -Dataclass <-> Arrow struct encoding for Orcapod. - -Encodes Python dataclasses as Arrow structs with a ``__type`` sentinel field -carrying the fully-qualified class name. Decoding uses a three-tier fallback: -import -> registry -> synthesize. -""" - -from __future__ import annotations - -import dataclasses -import importlib -import logging -import re -import typing -from typing import TYPE_CHECKING, Any - -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import pyarrow as pa -else: - pa = LazyModule("pyarrow") - -logger = logging.getLogger(__name__) - -DATACLASS_TYPE_FIELD = "__type" -DATACLASS_TYPE_PREFIX = "dataclass:" - -# Validates fully-qualified class names like "my_module.sub.MyClass" -_FQCN_RE = re.compile(r"^[A-Za-z_]\w*(\.[A-Za-z_]\w*)+$") - -# Process-global registry for tier-2 reconstruction. -# Populated via register_dataclass(); persists for the process lifetime. -_DATACLASS_REGISTRY: dict[str, type] = {} - - -def register_dataclass(cls: type) -> type: - """Register a dataclass for tier-2 reconstruction by fully-qualified name. - - Can be used as a class decorator or called directly. Returns ``cls`` - unchanged so it works transparently as a decorator. - - Args: - cls: A Python dataclass type to register. - - Returns: - The same ``cls`` that was passed in. - """ - key = f"{cls.__module__}.{cls.__qualname__}" - _DATACLASS_REGISTRY[key] = cls - return cls -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py::test_constants tests/test_semantic_types/test_dataclass_encoding.py::test_register_explicit tests/test_semantic_types/test_dataclass_encoding.py::test_register_returns_class tests/test_semantic_types/test_dataclass_encoding.py::test_register_as_decorator -v -``` - -Expected: 4 passed - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/dataclass_encoding.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): add module skeleton with constants and registry (ENG-555)" -``` - ---- - -### Task 2: `has_dataclass_type_sentinel` helper - -**Files:** -- Modify: `src/orcapod/semantic_types/dataclass_encoding.py` -- Modify: `tests/test_semantic_types/test_dataclass_encoding.py` - -- [ ] **Step 1: Add tests** - -Append to `tests/test_semantic_types/test_dataclass_encoding.py`: - -```python -import pyarrow as pa -from orcapod.semantic_types.dataclass_encoding import has_dataclass_type_sentinel - - -def test_sentinel_large_string(): - t = pa.struct([pa.field("__type", pa.large_string()), pa.field("a", pa.int64())]) - assert has_dataclass_type_sentinel(t) is True - - -def test_sentinel_string_compat(): - # older Arrow versions wrote pa.string() instead of pa.large_string() - t = pa.struct([pa.field("__type", pa.string()), pa.field("a", pa.int64())]) - assert has_dataclass_type_sentinel(t) is True - - -def test_sentinel_missing_field(): - t = pa.struct([pa.field("a", pa.int64()), pa.field("b", pa.large_string())]) - assert has_dataclass_type_sentinel(t) is False - - -def test_sentinel_non_struct(): - assert has_dataclass_type_sentinel(pa.int64()) is False -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "sentinel" -v -``` - -Expected: `ImportError: cannot import name 'has_dataclass_type_sentinel'` - -- [ ] **Step 3: Implement `has_dataclass_type_sentinel`** - -Append to `src/orcapod/semantic_types/dataclass_encoding.py` (after `register_dataclass`): - -```python -def has_dataclass_type_sentinel(arrow_type: pa.DataType) -> bool: - """Return ``True`` if ``arrow_type`` is a struct with a ``__type`` string field. - - Accepts both ``pa.large_string()`` and ``pa.string()`` for compatibility - with data written by older Arrow versions. - - Args: - arrow_type: Any PyArrow data type. - - Returns: - True if ``arrow_type`` is a struct containing a ``__type: (large_)string`` - field. - """ - if not pa.types.is_struct(arrow_type): - return False - for i in range(arrow_type.num_fields): - field = arrow_type.field(i) - if field.name == DATACLASS_TYPE_FIELD: - return pa.types.is_large_string(field.type) or pa.types.is_string(field.type) - return False -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "sentinel" -v -``` - -Expected: 4 passed - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/dataclass_encoding.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): add has_dataclass_type_sentinel helper (ENG-555)" -``` - ---- - -### Task 3: `dataclass_to_arrow_struct_type` encoder - -**Files:** -- Modify: `src/orcapod/semantic_types/dataclass_encoding.py` -- Modify: `tests/test_semantic_types/test_dataclass_encoding.py` - -The `converter` argument is a `UniversalTypeConverter` instance. These tests use a plain -(not-yet-dataclass-wired) converter — it handles basic field types `int`, `str`, `float`. -Nested dataclass round-trips are tested in Task 6 once UTC is fully wired. - -- [ ] **Step 1: Add tests** - -Append to `tests/test_semantic_types/test_dataclass_encoding.py`: - -```python -from orcapod.semantic_types.universal_converter import UniversalTypeConverter -from orcapod.semantic_types.dataclass_encoding import dataclass_to_arrow_struct_type - - -def test_struct_type_basic_fields(): - @dataclasses.dataclass - class _Point: - x: int - y: float - - converter = UniversalTypeConverter() - result = dataclass_to_arrow_struct_type(_Point, converter) - - assert pa.types.is_struct(result) - # __type must be the first field - assert result[0].name == "__type" - assert result[0].type == pa.large_string() - assert result.field("x").type == pa.int64() - assert result.field("y").type == pa.float64() - - -def test_struct_type_string_field(): - @dataclasses.dataclass - class _Named: - name: str - - converter = UniversalTypeConverter() - result = dataclass_to_arrow_struct_type(_Named, converter) - assert result.field("name").type == pa.large_string() - - -def test_struct_type_non_dataclass_raises(): - converter = UniversalTypeConverter() - with pytest.raises(TypeError, match="not a dataclass"): - dataclass_to_arrow_struct_type(int, converter) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "struct_type" -v -``` - -Expected: `ImportError: cannot import name 'dataclass_to_arrow_struct_type'` - -- [ ] **Step 3: Implement `dataclass_to_arrow_struct_type`** - -Append to `src/orcapod/semantic_types/dataclass_encoding.py`: - -```python -def dataclass_to_arrow_struct_type( - cls: type, - converter: Any, -) -> pa.StructType: - """Derive the Arrow struct type for a dataclass class. - - The resulting struct has ``__type: large_string`` as its first field, - followed by one field per dataclass field. Field types are resolved via - ``converter`` (a ``UniversalTypeConverter``), so nested dataclasses - produce nested structs automatically once the converter has the dataclass - branch wired in. - - Args: - cls: A Python dataclass type. - converter: A ``UniversalTypeConverter`` instance used for field type - resolution. - - Returns: - A ``pa.StructType`` with ``__type`` as the first field. - - Raises: - TypeError: If ``cls`` is not a dataclass type. - """ - if not dataclasses.is_dataclass(cls) or not isinstance(cls, type): - raise TypeError(f"{cls!r} is not a dataclass type") - - hints = typing.get_type_hints(cls) - fields: list[pa.Field] = [pa.field(DATACLASS_TYPE_FIELD, pa.large_string())] - for f in dataclasses.fields(cls): - arrow_type = converter.python_type_to_arrow_type(hints[f.name]) - fields.append(pa.field(f.name, arrow_type)) - return pa.struct(fields) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "struct_type" -v -``` - -Expected: 3 passed - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/dataclass_encoding.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): add dataclass_to_arrow_struct_type (ENG-555)" -``` - ---- - -### Task 4: `dataclass_to_struct_dict` encoder - -**Files:** -- Modify: `src/orcapod/semantic_types/dataclass_encoding.py` -- Modify: `tests/test_semantic_types/test_dataclass_encoding.py` - -- [ ] **Step 1: Add tests** - -Append to `tests/test_semantic_types/test_dataclass_encoding.py`: - -```python -from orcapod.semantic_types.dataclass_encoding import dataclass_to_struct_dict - - -def _build_field_converters(cls: type, converter: UniversalTypeConverter) -> dict: - """Helper: build per-field Arrow-value converters for a dataclass.""" - hints = typing.get_type_hints(cls) - return { - f.name: converter.get_python_to_arrow_converter(hints[f.name]) - for f in dataclasses.fields(cls) - } - - -def test_struct_dict_simple(): - @dataclasses.dataclass - class _Box: - width: int - label: str - - converter = UniversalTypeConverter() - field_converters = _build_field_converters(_Box, converter) - obj = _Box(width=10, label="big") - result = dataclass_to_struct_dict(obj, field_converters) - - fqcn = f"{_Box.__module__}.{_Box.__qualname__}" - assert result[DATACLASS_TYPE_FIELD] == f"dataclass:{fqcn}" - assert result["width"] == 10 - assert result["label"] == "big" - - -def test_struct_dict_type_error_on_class(): - # Passing a class (not instance) should raise - with pytest.raises(TypeError, match="not a dataclass instance"): - dataclass_to_struct_dict(_Simple, {}) - - -def test_struct_dict_type_error_on_non_dataclass(): - with pytest.raises(TypeError, match="not a dataclass instance"): - dataclass_to_struct_dict(42, {}) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "struct_dict" -v -``` - -Expected: `ImportError: cannot import name 'dataclass_to_struct_dict'` - -- [ ] **Step 3: Implement `dataclass_to_struct_dict`** - -Append to `src/orcapod/semantic_types/dataclass_encoding.py`: - -```python -def dataclass_to_struct_dict( - obj: Any, - field_converters: dict[str, Any], -) -> dict[str, Any]: - """Encode a dataclass instance to an Arrow-compatible struct dict. - - Args: - obj: A dataclass instance to encode. - field_converters: Pre-built per-field converter callables keyed by - field name. Build these once per type at converter-creation time - and reuse per row to avoid repeated type dispatch. - - Returns: - A dict with ``__type`` as the first key followed by encoded field values. - - Raises: - TypeError: If ``obj`` is not a dataclass instance (e.g. a class itself - or a non-dataclass value). - """ - # dataclasses.is_dataclass() returns True for both classes and instances; - # isinstance(obj, type) distinguishes: True for classes, False for instances. - if not dataclasses.is_dataclass(obj) or isinstance(obj, type): - raise TypeError(f"{obj!r} is not a dataclass instance") - - cls = type(obj) - type_str = f"{DATACLASS_TYPE_PREFIX}{cls.__module__}.{cls.__qualname__}" - result: dict[str, Any] = {DATACLASS_TYPE_FIELD: type_str} - for f in dataclasses.fields(cls): - value = getattr(obj, f.name) - converter_fn = field_converters.get(f.name, lambda v: v) - result[f.name] = converter_fn(value) - return result -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "struct_dict" -v -``` - -Expected: 3 passed - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/dataclass_encoding.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): add dataclass_to_struct_dict encoder (ENG-555)" -``` - ---- - -### Task 5: Three-tier decoder — `struct_dict_to_dataclass` - -**Files:** -- Modify: `src/orcapod/semantic_types/dataclass_encoding.py` -- Modify: `tests/test_semantic_types/test_dataclass_encoding.py` - -Tests use `unittest.mock.patch` to control import behaviour without filesystem dependencies. - -- [ ] **Step 1: Add tests** - -Append to `tests/test_semantic_types/test_dataclass_encoding.py`: - -```python -from unittest.mock import MagicMock, patch -from orcapod.semantic_types.dataclass_encoding import struct_dict_to_dataclass - - -@dataclasses.dataclass -class _TierOne: - value: int - - -def test_tier1_import(): - """Tier 1: class is importable via importlib.""" - fqcn = f"{_TierOne.__module__}.{_TierOne.__qualname__}" - struct_dict = { - "__type": f"dataclass:{fqcn}", - "value": 7, - } - field_converters = {"value": lambda v: v} - cache: dict = {} - - # Patch importlib so tier 1 returns _TierOne - mock_module = MagicMock() - mock_module.test_dataclass_encoding = MagicMock() - # The FQCN splits as module=, attr= - module_path, _, class_attr = fqcn.rpartition(".") - with patch("importlib.import_module") as mock_import: - mock_mod = MagicMock() - setattr(mock_mod, class_attr, _TierOne) - mock_import.return_value = mock_mod - - result = struct_dict_to_dataclass(struct_dict, field_converters, cache) - - assert isinstance(result, _TierOne) - assert result.value == 7 - # Cache should be populated - assert cache[fqcn] is _TierOne - - -def test_tier1_cache_hit(): - """Tier 1: cache hit skips importlib entirely.""" - fqcn = "some.module.SomeClass" - cache = {fqcn: _TierOne} - struct_dict = {"__type": f"dataclass:{fqcn}", "value": 3} - field_converters = {"value": lambda v: v} - - with patch("importlib.import_module") as mock_import: - result = struct_dict_to_dataclass(struct_dict, field_converters, cache) - mock_import.assert_not_called() - - assert isinstance(result, _TierOne) - assert result.value == 3 - - -def test_tier2_registry(): - """Tier 2: importlib fails, class found in registry.""" - @dataclasses.dataclass - class _RegClass: - score: float - - fqcn = "fake.module.RegClass" - _DATACLASS_REGISTRY[fqcn] = _RegClass - - struct_dict = {"__type": f"dataclass:{fqcn}", "score": 9.5} - field_converters = {"score": lambda v: v} - cache: dict = {} - - with patch("importlib.import_module", side_effect=ImportError("no module")): - result = struct_dict_to_dataclass(struct_dict, field_converters, cache) - - assert isinstance(result, _RegClass) - assert result.score == 9.5 - assert cache[fqcn] is _RegClass - - -def test_tier3_synthesize(): - """Tier 3: neither importable nor registered — synthesize a dataclass.""" - fqcn = "totally.unknown.Ghost" - struct_dict = {"__type": f"dataclass:{fqcn}", "name": "phantom", "age": 99} - field_converters = {"name": lambda v: v, "age": lambda v: v} - cache: dict = {} - - with patch("importlib.import_module", side_effect=ImportError("no module")): - result = struct_dict_to_dataclass(struct_dict, field_converters, cache) - - assert dataclasses.is_dataclass(result) - assert result.name == "phantom" # type: ignore[attr-defined] - assert result.age == 99 # type: ignore[attr-defined] - # Synthesized class cached under fqcn for future rows - assert fqcn in cache - - -def test_missing_type_field_tier3(): - """Struct without __type falls through to tier 3 silently.""" - struct_dict = {"value": 42} - field_converters = {"value": lambda v: v} - cache: dict = {} - - result = struct_dict_to_dataclass(struct_dict, field_converters, cache) - - assert dataclasses.is_dataclass(result) - assert result.value == 42 # type: ignore[attr-defined] - # No cache entry — no valid fqcn to cache under - assert len(cache) == 0 - - -def test_malformed_type_field_tier3(): - """Invalid __type format (fails regex) falls through to tier 3.""" - struct_dict = {"__type": "not-valid!!!", "x": 1} - field_converters = {"x": lambda v: v} - cache: dict = {} - - result = struct_dict_to_dataclass(struct_dict, field_converters, cache) - - assert dataclasses.is_dataclass(result) - assert result.x == 1 # type: ignore[attr-defined] -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "tier" -v -``` - -Expected: `ImportError: cannot import name 'struct_dict_to_dataclass'` - -- [ ] **Step 3: Implement `struct_dict_to_dataclass`** - -Append to `src/orcapod/semantic_types/dataclass_encoding.py`: - -```python -def struct_dict_to_dataclass( - struct_dict: dict[str, Any], - field_converters: dict[str, Any], - lookup_cache: dict[str, type], -) -> Any: - """Decode an Arrow struct dict to a Python dataclass instance. - - Uses a three-tier fallback: - - 1. **Import** — ``importlib``-import the class from its fully-qualified name. - 2. **Registry** — look up the FQCN in the process-global ``_DATACLASS_REGISTRY``. - 3. **Synthesize** — create a throwaway dataclass with ``dataclasses.make_dataclass`` - matching the struct's field names (all fields typed as ``Any``). - - Tier 3 never raises. A ``lookup_cache`` (keyed by FQCN) amortises repeated - resolution across rows in the same read operation. - - Args: - struct_dict: Arrow struct row dict as produced by ``pa.Table.to_pylist()``. - field_converters: Per-field Arrow->Python converter callables (keyed by - field name, excluding ``__type``). - lookup_cache: Mutable dict used as a per-read cache. Pass the same dict - for all rows in a read operation; clear between operations if needed. - - Returns: - A dataclass instance (real or synthesized) with field values set. - """ - type_str = struct_dict.get(DATACLASS_TYPE_FIELD) - - fqcn: str | None = None - class_name = "SynthesizedDataclass" - - if type_str and isinstance(type_str, str) and type_str.startswith(DATACLASS_TYPE_PREFIX): - candidate = type_str[len(DATACLASS_TYPE_PREFIX):] - if _FQCN_RE.match(candidate): - fqcn = candidate - class_name = fqcn.rsplit(".", 1)[-1] - else: - logger.warning( - "struct_dict_to_dataclass: invalid __type value %r — falling back to tier 3", - type_str, - ) - - cls: type | None = None - - if fqcn is not None: - # Check lookup cache first (amortises tiers 1-3 across rows) - if fqcn in lookup_cache: - cls = lookup_cache[fqcn] - else: - # Tier 1: import - module_path, _, class_attr = fqcn.rpartition(".") - try: - module = importlib.import_module(module_path) - cls = getattr(module, class_attr) - lookup_cache[fqcn] = cls - except (ImportError, AttributeError) as exc: - logger.debug( - "struct_dict_to_dataclass: tier 1 import failed for %r: %s", - fqcn, exc, - ) - - # Tier 2: registry - if cls is None: - cls = _DATACLASS_REGISTRY.get(fqcn) - if cls is not None: - lookup_cache[fqcn] = cls - - # Tier 3: synthesize (fqcn valid but unresolvable) - if cls is None: - field_names = [k for k in struct_dict if k != DATACLASS_TYPE_FIELD] - cls = dataclasses.make_dataclass( - class_name, [(name, typing.Any) for name in field_names] - ) - lookup_cache[fqcn] = cls - else: - # No valid fqcn — tier 3 with no caching (no stable key) - field_names = [k for k in struct_dict if k != DATACLASS_TYPE_FIELD] - cls = dataclasses.make_dataclass( - class_name, [(name, typing.Any) for name in field_names] - ) - - # Instantiate: apply field converters, skip the __type sentinel - data_kwargs: dict[str, Any] = {} - for key, value in struct_dict.items(): - if key == DATACLASS_TYPE_FIELD: - continue - converter_fn = field_converters.get(key, lambda v: v) - data_kwargs[key] = converter_fn(value) if value is not None else None - - return cls(**data_kwargs) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "tier or missing or malformed" -v -``` - -Expected: 7 passed - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/dataclass_encoding.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): add three-tier struct_dict_to_dataclass decoder (ENG-555)" -``` - ---- - -### Task 6: Wire into `UniversalTypeConverter` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Modify: `tests/test_semantic_types/test_dataclass_encoding.py` - -- [ ] **Step 1: Add tests (simple and nested round-trips through UTC)** - -Append to `tests/test_semantic_types/test_dataclass_encoding.py`: - -```python -from orcapod.semantic_types.universal_converter import UniversalTypeConverter as _UTC - - -def test_utc_simple_round_trip(): - """Full encode->decode round-trip through UniversalTypeConverter.""" - @dataclasses.dataclass - class _Color: - r: int - g: int - b: int - - converter = _UTC() - arrow_type = converter.python_type_to_arrow_type(_Color) - assert has_dataclass_type_sentinel(arrow_type) - - obj = _Color(r=255, g=128, b=0) - encode = converter.get_python_to_arrow_converter(_Color) - encoded = encode(obj) - assert encoded["__type"] == f"dataclass:{_Color.__module__}.{_Color.__qualname__}" - - decode = converter.get_arrow_to_python_converter(arrow_type) - with patch("importlib.import_module") as mock_import: - mock_mod = MagicMock() - setattr(mock_mod, "_Color", _Color) - mock_import.return_value = mock_mod - result = decode(encoded) - - assert isinstance(result, _Color) - assert result.r == 255 and result.g == 128 and result.b == 0 - - -def test_utc_nested_round_trip(): - """Nested dataclass encodes and decodes recursively.""" - @dataclasses.dataclass - class _Inner: - y: float - - @dataclasses.dataclass - class _Outer: - x: int - inner: _Inner - - converter = _UTC() - arrow_type = converter.python_type_to_arrow_type(_Outer) - - # Nested struct: inner field should itself be a __type-bearing struct - inner_arrow = arrow_type.field("inner").type - assert has_dataclass_type_sentinel(inner_arrow) - - obj = _Outer(x=1, inner=_Inner(y=3.14)) - encode = converter.get_python_to_arrow_converter(_Outer) - encoded = encode(obj) - - assert encoded["inner"]["__type"] == f"dataclass:{_Inner.__module__}.{_Inner.__qualname__}" - assert encoded["inner"]["y"] == 3.14 - - decode = converter.get_arrow_to_python_converter(arrow_type) - - inner_fqcn = f"{_Inner.__module__}.{_Inner.__qualname__}" - outer_fqcn = f"{_Outer.__module__}.{_Outer.__qualname__}" - inner_attr = inner_fqcn.rpartition(".")[2] - outer_attr = outer_fqcn.rpartition(".")[2] - - with patch("importlib.import_module") as mock_import: - def fake_import(module_path): - mod = MagicMock() - setattr(mod, inner_attr, _Inner) - setattr(mod, outer_attr, _Outer) - return mod - mock_import.side_effect = fake_import - result = decode(encoded) - - assert isinstance(result, _Outer) - assert result.x == 1 - assert isinstance(result.inner, _Inner) - assert result.inner.y == 3.14 - - -def test_utc_clear_cache_clears_dataclass_cache(): - converter = _UTC() - - @dataclasses.dataclass - class _Temp: - n: int - - fqcn = f"{_Temp.__module__}.{_Temp.__qualname__}" - converter._dataclass_lookup_cache[fqcn] = _Temp - converter.clear_cache() - assert fqcn not in converter._dataclass_lookup_cache -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "utc" -v -``` - -Expected: The round-trip tests fail because `UniversalTypeConverter` does not yet handle dataclasses. - -- [ ] **Step 3: Modify `universal_converter.py`** - -**3a.** Add imports at the top of `src/orcapod/semantic_types/universal_converter.py`, after the existing imports: - -```python -import dataclasses - -from orcapod.semantic_types.dataclass_encoding import ( - DATACLASS_TYPE_FIELD, - dataclass_to_arrow_struct_type, - dataclass_to_struct_dict, - has_dataclass_type_sentinel, - struct_dict_to_dataclass, -) -``` - -**3b.** In `UniversalTypeConverter.__init__`, add one line after the existing cache dicts: - -```python -self._dataclass_lookup_cache: dict[str, type] = {} -``` - -**3c.** In `_convert_python_to_arrow`, insert before the `if origin is None:` block: - -```python -# Dataclass types → struct with __type sentinel -if dataclasses.is_dataclass(python_type) and isinstance(python_type, type): - return dataclass_to_arrow_struct_type(python_type, self) -``` - -**3d.** In `_create_python_to_arrow_converter`, insert before the `origin = get_origin(python_type)` line: - -```python -# Dataclass instances → struct dict with __type sentinel -if dataclasses.is_dataclass(python_type) and isinstance(python_type, type): - hints = typing.get_type_hints(python_type) - field_converters = { - f.name: self.get_python_to_arrow_converter(hints[f.name]) - for f in dataclasses.fields(python_type) - } - return lambda obj: dataclass_to_struct_dict(obj, field_converters) -``` - -**3e.** In `_convert_arrow_to_python`, inside the `elif pa.types.is_struct(arrow_type):` block, insert before the check for heterogeneous tuples: - -```python -# Dataclass structs: actual type is resolved per-row at decode time via __type value -if has_dataclass_type_sentinel(arrow_type): - return Any -``` - -**3f.** In `_create_arrow_to_python_converter`, inside the `elif pa.types.is_struct(arrow_type):` block, insert before the `if python_type is tuple ...` check: - -```python -# Dataclass structs: per-row dispatch via __type value -if has_dataclass_type_sentinel(arrow_type): - field_converters = { - field.name: self.get_arrow_to_python_converter(field.type) - for field in arrow_type - if field.name != DATACLASS_TYPE_FIELD - } - cache = self._dataclass_lookup_cache - return lambda d: struct_dict_to_dataclass(d, field_converters, cache) -``` - -**3g.** In `clear_cache`, add one line at the end: - -```python -self._dataclass_lookup_cache.clear() -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "utc" -v -``` - -Expected: 3 passed - -- [ ] **Step 5: Run the full test suite to check for regressions** - -```bash -uv run pytest tests/test_semantic_types/ -v -``` - -Expected: all existing tests still pass - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): wire dataclass encoding into UniversalTypeConverter (ENG-555)" -``` - ---- - -### Task 7: Polymorphic decode and public API - -**Files:** -- Modify: `src/orcapod/__init__.py` -- Modify: `tests/test_semantic_types/test_dataclass_encoding.py` - -- [ ] **Step 1: Add polymorphic decode test and Parquet integration test** - -Append to `tests/test_semantic_types/test_dataclass_encoding.py`: - -```python -import tempfile -import os - - -def test_polymorphic_decode(): - """Two rows with different __type values each decode to their own class.""" - @dataclasses.dataclass - class _Cat: - name: str - - @dataclasses.dataclass - class _Dog: - name: str - - cat_fqcn = f"{_Cat.__module__}.{_Cat.__qualname__}" - dog_fqcn = f"{_Dog.__module__}.{_Dog.__qualname__}" - - # Both have the same Arrow schema (name: large_string) plus __type - arrow_type = pa.struct([ - pa.field("__type", pa.large_string()), - pa.field("name", pa.large_string()), - ]) - converter = _UTC() - decode = converter.get_arrow_to_python_converter(arrow_type) - - cat_attr = cat_fqcn.rpartition(".")[2] - dog_attr = dog_fqcn.rpartition(".")[2] - - with patch("importlib.import_module") as mock_import: - def fake_import(module_path): - mod = MagicMock() - setattr(mod, cat_attr, _Cat) - setattr(mod, dog_attr, _Dog) - return mod - mock_import.side_effect = fake_import - - row0 = decode({"__type": f"dataclass:{cat_fqcn}", "name": "Whiskers"}) - row1 = decode({"__type": f"dataclass:{dog_fqcn}", "name": "Rex"}) - - assert isinstance(row0, _Cat) and row0.name == "Whiskers" - assert isinstance(row1, _Dog) and row1.name == "Rex" - - -@pytest.mark.integration -def test_parquet_round_trip(): - """Full round-trip: python_dicts_to_arrow_table -> Parquet -> arrow_table_to_python_dicts.""" - import pyarrow.parquet as pq - from orcapod.semantic_types.universal_converter import UniversalTypeConverter as _UTC2 - - @dataclasses.dataclass - class _Record: - score: float - label: str - - converter = _UTC2() - - python_dicts = [ - {"rec": _Record(score=0.9, label="good")}, - {"rec": _Record(score=0.1, label="bad")}, - ] - from orcapod.types import Schema - python_schema = Schema({"rec": _Record}) - table = converter.python_dicts_to_arrow_table(python_dicts, python_schema=python_schema) - - with tempfile.TemporaryDirectory() as tmpdir: - path = os.path.join(tmpdir, "test.parquet") - pq.write_table(table, path) - loaded = pq.read_table(path) - - rec_fqcn = f"{_Record.__module__}.{_Record.__qualname__}" - rec_attr = rec_fqcn.rpartition(".")[2] - - with patch("importlib.import_module") as mock_import: - mod = MagicMock() - setattr(mod, rec_attr, _Record) - mock_import.return_value = mod - results = converter.arrow_table_to_python_dicts(loaded) - - assert len(results) == 2 - assert isinstance(results[0]["rec"], _Record) - assert results[0]["rec"].score == 0.9 - assert results[0]["rec"].label == "good" - assert isinstance(results[1]["rec"], _Record) - assert results[1]["rec"].score == 0.1 - assert results[1]["rec"].label == "bad" -``` - -- [ ] **Step 2: Run tests to verify they fail (or mark correctly)** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -k "polymorphic or parquet" -v -``` - -Expected: `test_polymorphic_decode` passes, `test_parquet_round_trip` fails (schema issue — `Schema` doesn't know `_Record` type yet until we wire it in a moment) - -- [ ] **Step 3: Expose `register_dataclass` in `orcapod/__init__.py`** - -Edit `src/orcapod/__init__.py` to add the import and export: - -```python -from .config import OrcapodConfig -from .core.function_pod import ( - FunctionPod, - function_pod, -) -from .core.nodes.source_node import SourceNode -from .pipeline import Pipeline, PipelineJob -from .semantic_types.dataclass_encoding import register_dataclass - -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -__all__ = [ - "OrcapodConfig", - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceNode", - "register_dataclass", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", -] -``` - -- [ ] **Step 4: Run all new tests** - -```bash -uv run pytest tests/test_semantic_types/test_dataclass_encoding.py -v -``` - -Expected: all tests pass (the Parquet integration test should now pass too) - -- [ ] **Step 5: Run full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -v -``` - -Expected: all existing tests pass - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/__init__.py tests/test_semantic_types/test_dataclass_encoding.py -git commit -m "feat(dataclass): expose register_dataclass in public API, add integration tests (ENG-555)" -``` - ---- - -## Final Verification - -Run the complete test suite one last time: - -```bash -uv run pytest tests/ -v -``` - -All tests should pass. Then push the branch: - -```bash -git push -u origin eywalker/eng-555-add-dataclass-support-to-orcapod-via-arrow-struct-encoding -``` - ---- - -## Self-Review Checklist (completed inline) - -- **Spec coverage:** - - `__type` sentinel field: Task 3 (`dataclass_to_arrow_struct_type`) - - `dataclass:` prefix value: Task 4 (`dataclass_to_struct_dict`) - - Tier 1 import: Task 5 `test_tier1_import` - - Tier 2 registry + `register_dataclass`: Tasks 1 + 5 `test_tier2_registry` - - Tier 3 synthesize: Task 5 `test_tier3_synthesize` - - Lookup cache: Task 5 `test_tier1_cache_hit` + Task 6 `test_utc_clear_cache_clears_dataclass_cache` - - Nested dataclasses: Task 6 `test_utc_nested_round_trip` - - Polymorphic decode: Task 7 `test_polymorphic_decode` - - Parquet round-trip: Task 7 `test_parquet_round_trip` - - Public API `orcapod.register_dataclass`: Task 7 + `test_register_as_decorator` - - Malformed `__type` fallback: Task 5 `test_malformed_type_field_tier3` - - Missing `__type` fallback: Task 5 `test_missing_type_field_tier3` - - `TypeError` on non-dataclass: Task 4 + Task 3 - -- **No placeholders:** Verified — all steps contain complete code. - -- **Type consistency:** `field_converters: dict[str, Any]` used consistently across Tasks 4-6. `lookup_cache: dict[str, type]` consistent across Tasks 5-6. `converter: Any` (typed as `UniversalTypeConverter` in practice) consistent in Tasks 3-6. diff --git a/superpowers/plans/2026-06-02-eng-377-function-node-fetch-join-refactor.md b/superpowers/plans/2026-06-02-eng-377-function-node-fetch-join-refactor.md deleted file mode 100644 index b2ed4e831..000000000 --- a/superpowers/plans/2026-06-02-eng-377-function-node-fetch-join-refactor.md +++ /dev/null @@ -1,864 +0,0 @@ -# ENG-377: FunctionJobNode Fetch-Join Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Extract the duplicated "fetch both DBs → content-hash-filter → inner-join → restore nullability" logic from `get_all_records` and `_load_cached_entries` into a single private helper `_fetch_joined_records`, eliminating the maintenance risk of two diverging copies. - -**Architecture:** Add a module-level `_PIPELINE_ENTRY_ID_COL` constant and a `_JoinedRecords` NamedTuple. Add `_fetch_joined_records(entry_ids=None) -> _JoinedRecords | None` to `FunctionJobNode`. Refactor `_load_cached_entries` and `get_all_records` to delegate the fetch-and-join work to it. Update all four method docstrings to clarify the invocation chain. - -**Tech Stack:** Python, PyArrow (`pa`), Polars (`pl`), `NamedTuple` from `typing`. All commands via `uv run`. - -**Spec:** `superpowers/specs/2026-06-02-function-node-fetch-join-refactor-design.md` - ---- - -## File Structure - -| Action | Path | What changes | -|---|---|---| -| Modify | `src/orcapod/core/nodes/function_node.py` | Add `NamedTuple` import; add `_PIPELINE_ENTRY_ID_COL` constant; add `_JoinedRecords` NamedTuple; add `_fetch_joined_records`; refactor `_load_cached_entries`; refactor `get_all_records`; update 4 docstrings | -| Create | `tests/test_core/nodes/test_function_node_fetch_joined.py` | Unit tests for `_fetch_joined_records` | - ---- - -## Task 1: Create Feature Branch - -**Files:** none (git only) - -- [ ] **Step 1.1: Check out the branch** - -```bash -cd /home/kurouto/kurouto-jobs/711e18aa-7748-4686-8634-4f858cb0e9ed/orcapod-python -git checkout -b eywalker/eng-377-investigate-logic-redundancy-between-get_all_records-and -git branch --show-current -``` - -Expected output: `eywalker/eng-377-investigate-logic-redundancy-between-get_all_records-and` - ---- - -## Task 2: Write Failing Tests for `_fetch_joined_records` - -**Files:** -- Create: `tests/test_core/nodes/test_function_node_fetch_joined.py` - -The tests call `_fetch_joined_records` which does not exist yet. They should fail with `AttributeError: 'FunctionJobNode' object has no attribute '_fetch_joined_records'`. They also import `_PIPELINE_ENTRY_ID_COL` which does not exist yet, so the import itself will fail. Both are the correct "red" state. - -- [ ] **Step 2.1: Create the test file** - -```python -# tests/test_core/nodes/test_function_node_fetch_joined.py -"""Tests for FunctionJobNode._fetch_joined_records.""" - -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes.function_node import FunctionJobNode, _PIPELINE_ENTRY_ID_COL -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase - - -def double_value(value: int) -> int: - return value * 2 - - -@pytest.fixture -def node_without_db(): - """FunctionJobNode with no databases attached.""" - table = pa.table( - { - "key": pa.array(["a"], type=pa.large_string()), - "value": pa.array([1], type=pa.int64()), - } - ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - return FunctionJobNode(pod, src) - - -@pytest.fixture -def node_with_empty_db(): - """FunctionJobNode with databases attached but no data executed.""" - table = pa.table( - { - "key": pa.array(["a", "b"], type=pa.large_string()), - "value": pa.array([1, 2], type=pa.int64()), - } - ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - return FunctionJobNode( - pod, - src, - pipeline_database=InMemoryArrowDatabase(), - result_database=InMemoryArrowDatabase(), - ) - - -@pytest.fixture -def executed_node(node_with_empty_db): - """FunctionJobNode with databases attached and both input rows executed.""" - node = node_with_empty_db - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - return node - - -class TestFetchJoinedRecords: - def test_returns_none_when_no_db(self, node_without_db): - """Returns None when no databases are attached.""" - assert node_without_db._fetch_joined_records() is None - - def test_returns_none_when_db_fetch_returns_none(self, node_with_empty_db): - """Returns None when the pipeline DB returns None (no records written yet).""" - assert node_with_empty_db._fetch_joined_records() is None - - def test_returned_table_includes_pipeline_entry_id_column(self, executed_node): - """The returned table always contains __pipeline_entry_id.""" - result = executed_node._fetch_joined_records() - assert result is not None - assert _PIPELINE_ENTRY_ID_COL in result.table.column_names - - def test_taginfo_columns_present_in_result(self, executed_node): - """taginfo_columns in the returned NamedTuple is a non-empty tuple of strings.""" - result = executed_node._fetch_joined_records() - assert result is not None - assert isinstance(result.taginfo_columns, tuple) - assert len(result.taginfo_columns) > 0 - - def test_no_entry_ids_returns_all_rows(self, executed_node): - """Calling with entry_ids=None returns all executed rows.""" - result = executed_node._fetch_joined_records() - assert result is not None - assert result.table.num_rows == 2 - - def test_entry_ids_filter_narrows_rows(self, executed_node): - """Passing a single entry_id returns only that row.""" - node = executed_node - input_pairs = list(node._input_stream.iter_data()) - entry_id_0 = node.compute_pipeline_entry_id(input_pairs[0][0], input_pairs[0][1]) - - result = node._fetch_joined_records(entry_ids=[entry_id_0]) - assert result is not None - assert result.table.num_rows == 1 - assert result.table.column(_PIPELINE_ENTRY_ID_COL)[0].as_py() == entry_id_0 -``` - -- [ ] **Step 2.2: Run the tests and verify they fail at import** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_fetch_joined.py -v 2>&1 | head -30 -``` - -Expected: `ImportError` or `AttributeError` mentioning `_PIPELINE_ENTRY_ID_COL` or `_fetch_joined_records`. The tests must be red before proceeding. - ---- - -## Task 3: Implement `_PIPELINE_ENTRY_ID_COL`, `_JoinedRecords`, and `_fetch_joined_records` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` - -### Step 3.1 — Add `NamedTuple` to the `typing` import - -- [ ] **Step 3.1: Edit line 22 of `function_node.py`** - -Find: -```python -from typing import TYPE_CHECKING, Any, Literal, cast -``` - -Replace with: -```python -from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast -``` - -### Step 3.2 — Add the `_PIPELINE_ENTRY_ID_COL` module-level constant - -- [ ] **Step 3.2: Add constant after the `LazyModule` definitions (after line 68, before the `_executor_supports_concurrent` function)** - -Find: -```python -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - import pyarrow.compute as pc -else: - pa = LazyModule("pyarrow") - pc = LazyModule("pyarrow.compute") - pl = LazyModule("polars") - - -def _executor_supports_concurrent( -``` - -Replace with: -```python -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - import pyarrow.compute as pc -else: - pa = LazyModule("pyarrow") - pc = LazyModule("pyarrow.compute") - pl = LazyModule("polars") - -# Pipeline entry ID column name used when fetching records from the pipeline -# database. Always present in the table returned by _fetch_joined_records. -_PIPELINE_ENTRY_ID_COL = "__pipeline_entry_id" - - -def _executor_supports_concurrent( -``` - -### Step 3.3 — Add `_JoinedRecords` NamedTuple before `FunctionJobNode` - -- [ ] **Step 3.3: Locate the `FunctionJobNode` class definition (around line 684) and add `_JoinedRecords` immediately before it** - -Find the line that reads: -```python -class FunctionJobNode(FunctionNodeBase): -``` - -Insert the following block immediately before that line (one blank line separation): - -```python -class _JoinedRecords(NamedTuple): - """Internal result type returned by ``_fetch_joined_records``. - - Attributes: - table: The joined ``pa.Table``, always including a - ``__pipeline_entry_id`` column (the pipeline DB row key) and - ``DATA_RECORD_ID``. Does not have ``ColumnConfig`` filtering - applied — that is the caller's responsibility. - taginfo_columns: Column names from the pipeline database fetch, - captured before the join. Used by ``_load_cached_entries`` to - derive tag keys in the CACHE_ONLY (``_input_stream is None``) - fallback path, where the tag columns cannot be inferred from the - input stream and must be identified by exclusion from the taginfo - column set. - """ - - table: pa.Table - taginfo_columns: tuple[str, ...] - - -``` - -### Step 3.4 — Add `_fetch_joined_records` to `FunctionJobNode` - -- [ ] **Step 3.4: Add `_fetch_joined_records` inside `FunctionJobNode`, in the `# Cache-only helpers` section, immediately before `_load_cached_entries`** - -Find the block that starts with: -```python - # ------------------------------------------------------------------ - # Cache-only helpers (PLT-1156) - # ------------------------------------------------------------------ - - def _load_cached_entries( -``` - -Replace with: -```python - # ------------------------------------------------------------------ - # Cache-only helpers (PLT-1156) - # ------------------------------------------------------------------ - - def _fetch_joined_records( - self, - entry_ids: list[str] | None = None, - ) -> _JoinedRecords | None: - """Internal primitive: fetch both DBs, content-hash-filter, and inner-join. - - Fetches ``taginfo`` from the pipeline database with - ``_PIPELINE_ENTRY_ID_COL`` as the row-key column, fetches ``results`` - from the result database, applies ``_filter_by_content_hash``, and - inner-joins the two tables on ``DATA_RECORD_ID`` via polars. - - If ``entry_ids`` is provided, the polars DataFrame is filtered to - matching ``_PIPELINE_ENTRY_ID_COL`` values before conversion to Arrow, - avoiding a round-trip. - - Does NOT apply ``ColumnConfig`` column dropping (that is - ``get_all_records``'s job), convert rows to ``(tag, data)`` tuples - (that is ``_load_cached_entries``'s job), or touch the in-memory cache - (that is ``get_cached_results``'s job). - - Args: - entry_ids: If given, return only rows whose - ``_PIPELINE_ENTRY_ID_COL`` value is in this list. - If ``None``, return all rows. - - Returns: - A ``_JoinedRecords`` whose ``table`` always includes a - ``_PIPELINE_ENTRY_ID_COL`` column, or ``None`` when either - database is absent or either DB fetch returns ``None``. A 0-row - table is returned (not ``None``) when both fetches succeed but the - join finds no matching rows — callers check ``num_rows`` - themselves. - """ - if self._cached_function_pod is None or self._pipeline_database is None: - return None - - taginfo = self._pipeline_database.get_all_records( - self.node_identity_path, - record_id_column=_PIPELINE_ENTRY_ID_COL, - ) - results = self._cached_function_pod._result_database.get_all_records( - self._cached_function_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) - - if taginfo is None or results is None: - return None - - taginfo_columns = tuple(taginfo.column_names) - taginfo = self._filter_by_content_hash(taginfo) - taginfo_schema = taginfo.schema - results_schema = results.schema - - joined_df = pl.DataFrame(taginfo).join( - pl.DataFrame(results), - on=constants.DATA_RECORD_ID, - how="inner", - ) - if entry_ids is not None: - joined_df = joined_df.filter( - pl.col(_PIPELINE_ENTRY_ID_COL).is_in(entry_ids) - ) - joined = joined_df.to_arrow() - joined = arrow_utils.restore_schema_nullability( - joined, taginfo_schema, results_schema - ) - return _JoinedRecords(table=joined, taginfo_columns=taginfo_columns) - - def _load_cached_entries( -``` - -- [ ] **Step 3.5: Run only the new tests to verify they pass** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_fetch_joined.py -v -``` - -Expected: all 6 tests **PASS**. - -- [ ] **Step 3.6: Run the existing get_cached tests to confirm nothing broke** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_get_cached.py -v -``` - -Expected: all 5 tests **PASS**. - -- [ ] **Step 3.7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py \ - tests/test_core/nodes/test_function_node_fetch_joined.py -git commit -m "$(cat <<'EOF' -refactor(function_node): add _fetch_joined_records primitive and tests - -Extracts the fetch-and-join logic shared between get_all_records and -_load_cached_entries into a new _fetch_joined_records helper. Also adds -_PIPELINE_ENTRY_ID_COL module-level constant and _JoinedRecords NamedTuple. -Neither caller is wired up yet — that follows in the next two commits. - -Closes ENG-377 - -Co-Authored-By: Claude Sonnet 4.6 -EOF -)" -``` - ---- - -## Task 4: Refactor `_load_cached_entries` to Use `_fetch_joined_records` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines ~1442–1527) - -The current body of `_load_cached_entries` duplicates the fetch-and-join logic. Replace the entire body with a delegation to `_fetch_joined_records`, keeping only the tuple-conversion step. - -Note: `taginfo_columns` from `_JoinedRecords` is used in the `_input_stream is None` fallback path to derive `tag_keys` — this replaces the direct reference to `taginfo.column_names` that existed before the refactor. - -- [ ] **Step 4.1: Replace the body of `_load_cached_entries`** - -Find the entire current method body (from `def _load_cached_entries` through to just before `async def _async_execute_cache_only`): - -```python - def _load_cached_entries( - self, - entry_ids: list[str] | None = None, - ) -> "dict[str, tuple[TagProtocol, DataProtocol]]": - """Load (tag, data) pairs from pipeline DB + result DB. - - Args: - entry_ids: If provided, load only these specific entry IDs. - If ``None``, load all records for this node. - - Returns: - dict mapping entry_id → (tag, data). Empty dict when either - database is None, records are empty, or no rows match. - - Does NOT mutate ``_cached_output_datas``. - Callers merge via ``self._cached_output_datas.update(loaded)``. - """ - if self._cached_function_pod is None or self._pipeline_database is None: - return {} - - PIPELINE_ENTRY_ID_COL = "__pipeline_entry_id" - - taginfo = self._pipeline_database.get_all_records( - self.node_identity_path, - record_id_column=PIPELINE_ENTRY_ID_COL, - ) - results = self._cached_function_pod._result_database.get_all_records( - self._cached_function_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) - - if taginfo is None or results is None: - return {} - - taginfo = self._filter_by_content_hash(taginfo) - taginfo_schema = taginfo.schema - results_schema = results.schema - - joined_df = pl.DataFrame(taginfo).join( - pl.DataFrame(results), - on=constants.DATA_RECORD_ID, - how="inner", - ) - if entry_ids is not None: - joined_df = joined_df.filter( - pl.col(PIPELINE_ENTRY_ID_COL).is_in(entry_ids) - ) - joined = joined_df.to_arrow() - joined = arrow_utils.restore_schema_nullability( - joined, taginfo_schema, results_schema - ) - - if joined.num_rows == 0: - return {} - - # Derive tag keys: prefer input_stream when available; fall back to - # taginfo column exclusion for CACHE_ONLY / deserialized nodes. - if self._input_stream is not None: - tag_keys = self._input_stream.keys()[0] - else: - tag_keys = tuple( - c - for c in taginfo.column_names - if not c.startswith(constants.META_PREFIX) - and not c.startswith(constants.SOURCE_PREFIX) - and not c.startswith(constants.SYSTEM_TAG_PREFIX) - and c != PIPELINE_ENTRY_ID_COL - and c != constants.NODE_CONTENT_HASH_COL - ) - - # Drop internal columns (SOURCE_PREFIX is kept — ArrowTableStream needs it) - entry_ids_col = joined.column(PIPELINE_ENTRY_ID_COL).to_pylist() - drop_cols = [ - c - for c in joined.column_names - if c.startswith(constants.META_PREFIX) - or c == PIPELINE_ENTRY_ID_COL - or c == constants.NODE_CONTENT_HASH_COL - ] - data_table = joined.drop([c for c in drop_cols if c in joined.column_names]) - stream = ArrowTableStream(data_table, tag_columns=tag_keys) - - loaded: dict[str, tuple[TagProtocol, DataProtocol]] = {} - for eid, (tag, data) in zip(entry_ids_col, stream.iter_data()): - loaded[eid] = (tag, data) - return loaded -``` - -Replace with: - -```python - def _load_cached_entries( - self, - entry_ids: list[str] | None = None, - ) -> "dict[str, tuple[TagProtocol, DataProtocol]]": - """DB loader: fetch ``(tag, data)`` pairs from the pipeline and result databases. - - Calls ``_fetch_joined_records`` to obtain the raw joined table, then - converts each row into a ``(tag, data)`` tuple keyed by pipeline entry ID. - - If ``entry_ids`` is given, only those entries are fetched from DB. - If ``None``, all records for this node are loaded. - - Does NOT read from or write to the in-memory cache - (``_cached_output_datas``). Callers that want to populate the cache - must call ``self._cached_output_datas.update(loaded)`` themselves. - - Does NOT apply user-facing column filtering — see ``get_all_records`` - for that. - - Args: - entry_ids: If provided, load only these specific entry IDs. - If ``None``, load all records for this node. - - Returns: - dict mapping entry_id → ``(tag, data)``. Empty dict when either - database is absent, either DB fetch returns ``None``, or no rows - match after joining. - """ - fetched = self._fetch_joined_records(entry_ids=entry_ids) - if fetched is None or fetched.table.num_rows == 0: - return {} - - joined = fetched.table - - # Derive tag keys: prefer input_stream when available; fall back to - # taginfo column exclusion for CACHE_ONLY / deserialized nodes. - # taginfo_columns from _fetch_joined_records preserves the pipeline DB - # column names before joining, which is the correct exclusion set. - if self._input_stream is not None: - tag_keys = self._input_stream.keys()[0] - else: - tag_keys = tuple( - c - for c in fetched.taginfo_columns - if not c.startswith(constants.META_PREFIX) - and not c.startswith(constants.SOURCE_PREFIX) - and not c.startswith(constants.SYSTEM_TAG_PREFIX) - and c != _PIPELINE_ENTRY_ID_COL - and c != constants.NODE_CONTENT_HASH_COL - ) - - # Drop internal columns (SOURCE_PREFIX is kept — ArrowTableStream needs it) - entry_ids_col = joined.column(_PIPELINE_ENTRY_ID_COL).to_pylist() - drop_cols = [ - c - for c in joined.column_names - if c.startswith(constants.META_PREFIX) - or c == _PIPELINE_ENTRY_ID_COL - or c == constants.NODE_CONTENT_HASH_COL - ] - data_table = joined.drop([c for c in drop_cols if c in joined.column_names]) - stream = ArrowTableStream(data_table, tag_columns=tag_keys) - - loaded: dict[str, tuple[TagProtocol, DataProtocol]] = {} - for eid, (tag, data) in zip(entry_ids_col, stream.iter_data()): - loaded[eid] = (tag, data) - return loaded -``` - -- [ ] **Step 4.2: Run all existing cached-results tests** - -```bash -uv run pytest tests/test_core/nodes/ -v -``` - -Expected: all tests in `test_function_node_get_cached.py` and `test_function_node_fetch_joined.py` **PASS**. - -- [ ] **Step 4.3: Run the broader function_node integration tests** - -```bash -uv run pytest tests/test_core/function_pod/ tests/test_core/test_caching_integration.py -v -``` - -Expected: all tests **PASS**. - -- [ ] **Step 4.4: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "$(cat <<'EOF' -refactor(function_node): wire _load_cached_entries to _fetch_joined_records - -Removes the duplicated fetch-and-join block from _load_cached_entries. -The fallback CACHE_ONLY tag_keys derivation now uses -_JoinedRecords.taginfo_columns (the pre-join pipeline DB column names) -instead of a local taginfo reference, preserving correct semantics. - -Co-Authored-By: Claude Sonnet 4.6 -EOF -)" -``` - ---- - -## Task 5: Refactor `get_all_records` to Use `_fetch_joined_records` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines ~1355–1415) - -Two important notes: -1. The original `get_all_records` fetched taginfo without `record_id_column`, so `__pipeline_entry_id` was not in the table. Now it is. It starts with `__` (`META_PREFIX`), so the default column-config drop already removes it. But when `all_info=True`, meta columns are kept — so `__pipeline_entry_id` must be explicitly added to the always-drop list alongside `NODE_CONTENT_HASH_COL`. -2. The guard `if self._cached_function_pod is None` is dropped; `_fetch_joined_records` handles it. - -- [ ] **Step 5.1: Replace the body of `get_all_records`** - -Find the entire current `get_all_records` method: - -```python - def get_all_records( - self, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table | None: - """Return all computed results joined with their pipeline tag records. - - Args: - columns: Column configuration controlling which groups are included. - all_info: Shorthand to include all info columns. - - Returns: - A PyArrow table of joined results, or ``None`` if no database is - attached or no records exist. - """ - if self._cached_function_pod is None: - return None - - results = self._cached_function_pod._result_database.get_all_records( - self._cached_function_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) - taginfo = self._pipeline_database.get_all_records(self.node_identity_path) - - if results is None or taginfo is None: - return None - - taginfo = self._filter_by_content_hash(taginfo) - taginfo_schema = taginfo.schema - results_schema = results.schema - joined = ( - pl.DataFrame(taginfo) - .join(pl.DataFrame(results), on=constants.DATA_RECORD_ID, how="inner") - .to_arrow() - ) - joined = arrow_utils.restore_schema_nullability(joined, taginfo_schema, results_schema) - - column_config = ColumnConfig.handle_config(columns, all_info=all_info) - - drop_columns = [] - # Always drop the node content hash column — it is an internal - # row-level discriminator, not a user-facing column. - drop_columns.append(constants.NODE_CONTENT_HASH_COL) - if not column_config.meta and not column_config.all_info: - drop_columns.extend( - c for c in joined.column_names if c.startswith(constants.META_PREFIX) - ) - if not column_config.source and not column_config.all_info: - drop_columns.extend( - c for c in joined.column_names if c.startswith(constants.SOURCE_PREFIX) - ) - if not column_config.system_tags and not column_config.all_info: - drop_columns.extend( - c - for c in joined.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ) - if drop_columns: - joined = joined.drop([c for c in drop_columns if c in joined.column_names]) - - return joined if joined.num_rows > 0 else None -``` - -Replace with: - -```python - def get_all_records( - self, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table | None: - """Public table view: return all computed results joined with their pipeline records. - - Calls ``_fetch_joined_records`` to obtain the raw joined table, then - applies ``ColumnConfig``-driven column dropping to produce a - user-facing result. ``_PIPELINE_ENTRY_ID_COL`` and - ``NODE_CONTENT_HASH_COL`` are always dropped — they are internal - discriminator columns, not user-facing data. - - Does NOT populate the in-memory cache — see ``get_cached_results`` - for that. - - Args: - columns: Column configuration controlling which groups are - included. Accepts a ``ColumnConfig`` instance or a dict - shorthand (e.g. ``{"meta": True}``). - all_info: If ``True``, equivalent to enabling all column groups. - - Returns: - A ``pa.Table`` of joined results, or ``None`` if no database is - attached, either DB fetch returns ``None``, or the join produces - no rows. - """ - fetched = self._fetch_joined_records() - if fetched is None: - return None - - joined = fetched.table - column_config = ColumnConfig.handle_config(columns, all_info=all_info) - - # Always drop internal discriminator columns regardless of column_config. - # _PIPELINE_ENTRY_ID_COL starts with META_PREFIX so it is covered by - # the meta drop in the default case, but must be listed explicitly - # here so it is also dropped when all_info=True (which skips the - # meta-prefix sweep). - drop_columns = [constants.NODE_CONTENT_HASH_COL, _PIPELINE_ENTRY_ID_COL] - if not column_config.meta and not column_config.all_info: - drop_columns.extend( - c for c in joined.column_names if c.startswith(constants.META_PREFIX) - ) - if not column_config.source and not column_config.all_info: - drop_columns.extend( - c for c in joined.column_names if c.startswith(constants.SOURCE_PREFIX) - ) - if not column_config.system_tags and not column_config.all_info: - drop_columns.extend( - c - for c in joined.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ) - if drop_columns: - joined = joined.drop([c for c in drop_columns if c in joined.column_names]) - - return joined if joined.num_rows > 0 else None -``` - -- [ ] **Step 5.2: Run the full node and function_pod test suites** - -```bash -uv run pytest tests/test_core/nodes/ tests/test_core/function_pod/ tests/test_core/test_caching_integration.py -v -``` - -Expected: all tests **PASS**. - -- [ ] **Step 5.3: Run the broader pipeline tests to catch any regressions** - -```bash -uv run pytest tests/test_pipeline/ -v -``` - -Expected: all tests **PASS**. - -- [ ] **Step 5.4: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "$(cat <<'EOF' -refactor(function_node): wire get_all_records to _fetch_joined_records - -Removes the duplicated fetch-and-join block from get_all_records. -Adds _PIPELINE_ENTRY_ID_COL to the always-drop list so it is excluded -even when all_info=True (where the META_PREFIX sweep is skipped). - -Co-Authored-By: Claude Sonnet 4.6 -EOF -)" -``` - ---- - -## Task 6: Update Docstrings on `get_cached_results` and Final Test Run - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines ~1161–1194) - -`get_all_records` and `_load_cached_entries` already received updated docstrings in Tasks 4–5. This task updates `get_cached_results` (the public cache façade) to describe its role in the chain. `_fetch_joined_records` already has a full docstring from Task 3. - -- [ ] **Step 6.1: Update the `get_cached_results` docstring** - -Find: -```python - def get_cached_results( - self, entry_ids: list[str] - ) -> dict[str, tuple[TagProtocol, DataProtocol]]: - """Retrieve cached results for specific pipeline entry IDs. - - Checks in-memory cache first. Loads only truly missing entries from DB. - Add-only semantics: existing in-memory entries are never cleared or - overwritten (overwrite is safe since in-memory and DB entries for the - same entry_id are always semantically equivalent). - - Args: - entry_ids: Pipeline entry IDs to look up. - - Returns: - Mapping from entry_id to ``(tag, output_data)`` for found entries. - Empty dict if no DB is attached or no matches found. - """ -``` - -Replace with: -```python - def get_cached_results( - self, entry_ids: list[str] - ) -> dict[str, tuple[TagProtocol, DataProtocol]]: - """Public cache façade: return already-computed results for the given entry IDs. - - Serves hits directly from the in-memory cache (``_cached_output_datas``). - For IDs not yet cached, delegates to ``_load_cached_entries`` which calls - ``_fetch_joined_records`` to load from the pipeline and result databases. - Add-only semantics: existing in-memory entries are never cleared or - overwritten (safe because in-memory and DB entries for the same entry_id - are always semantically equivalent). - - Does NOT apply user-facing column filtering — see ``get_all_records`` - for that. - - Args: - entry_ids: Pipeline entry IDs to look up. - - Returns: - Mapping from entry_id to ``(tag, output_data)`` for found entries. - Empty dict if no DB is attached, ``entry_ids`` is empty, or no - matches are found. - """ -``` - -- [ ] **Step 6.2: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -30 -``` - -Expected: all tests **PASS**. If any fail, read the traceback and fix before committing. - -- [ ] **Step 6.3: Commit the docstring update** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "$(cat <<'EOF' -docs(function_node): update get_cached_results docstring to reflect invocation chain - -Clarifies that get_cached_results delegates cache misses to _load_cached_entries, -which in turn calls _fetch_joined_records. Completes ENG-377 docstring pass. - -Co-Authored-By: Claude Sonnet 4.6 -EOF -)" -``` - ---- - -## Self-Review - -**Spec coverage check:** - -| Spec requirement | Task that covers it | -|---|---| -| Extract `_fetch_joined_records` | Task 3 | -| Module-level `_PIPELINE_ENTRY_ID_COL` constant | Task 3 | -| `_JoinedRecords` NamedTuple with `table` + `taginfo_columns` | Task 3 | -| `_load_cached_entries` delegates to `_fetch_joined_records` | Task 4 | -| `get_all_records` delegates to `_fetch_joined_records` | Task 5 | -| Guard normalization (both callers drop individual guards) | Tasks 4 + 5 | -| `_PIPELINE_ENTRY_ID_COL` always dropped in `get_all_records` | Task 5 | -| Docstrings on all four methods | Tasks 3, 4, 5, 6 | -| New `test_function_node_fetch_joined.py` | Task 2 | -| Existing `TestGetCachedResults` tests pass | Verified in Tasks 3, 4, 5 | - -**Placeholder scan:** No TBDs, no "see above", no unresolved references. All code blocks are complete. ✓ - -**Type consistency:** `_JoinedRecords` defined in Task 3 with `.table: pa.Table` and `.taginfo_columns: tuple[str, ...]`. Both Task 4 (`fetched.table`, `fetched.taginfo_columns`) and Task 5 (`fetched.table`) use these exact attributes. `_PIPELINE_ENTRY_ID_COL` used consistently across Tasks 3, 4, and 5. ✓ diff --git a/superpowers/plans/2026-06-03-pipeline-test-migration.md b/superpowers/plans/2026-06-03-pipeline-test-migration.md deleted file mode 100644 index 279c24119..000000000 --- a/superpowers/plans/2026-06-03-pipeline-test-migration.md +++ /dev/null @@ -1,1133 +0,0 @@ -# Pipeline Test Migration to PipelineJob API — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Migrate all ~107 skipped `tests/test_pipeline/` tests from the old `Pipeline` API to the new `PipelineJob` API, restoring full regression coverage for the orchestration and observer subsystems. - -**Architecture:** Two small production changes (rename `ExecutorType` → `OrchestratorType`, add `orchestrator=` parameter to `PipelineJob.run()`) unlock the test migrations. Tests are migrated file-by-file in order of ascending complexity. No new features; no behavior changes. - -**Tech Stack:** Python 3.12+, pytest, uv, orcapod internal APIs (`PipelineJob`, `SyncPipelineOrchestrator`, `AsyncPipelineOrchestrator`, observer classes). - -**Spec:** `superpowers/specs/2026-06-03-pipeline-test-migration-design.md` - ---- - -## File Map - -| Action | File | Change | -|---|---|---| -| Modify | `src/orcapod/types.py` | Rename `ExecutorType` → `OrchestratorType`; `PipelineConfig.executor` → `.orchestrator` | -| Modify | `src/orcapod/pipeline/job.py` | Add `orchestrator=` param to `run()` | -| Modify | `tests/test_channels/test_channels.py` | Update `ExecutorType` → `OrchestratorType` references | -| Migrate | `tests/test_pipeline/test_sync_orchestrator.py` | 8 skipped classes | -| Migrate | `tests/test_pipeline/test_composite_observer.py` | 4 skipped classes | -| Migrate | `tests/test_pipeline/test_status_observer_integration.py` | 11 skipped classes | -| Migrate | `tests/test_pipeline/test_logging_observer_integration.py` | 9 skipped classes | -| Migrate | `tests/test_pipeline/test_orchestrator.py` | 8 skipped classes | -| Migrate | `tests/test_pipeline/test_graph_rendering.py` | 8 skipped classes | -| Migrate | `tests/test_pipeline/test_orchestrator_executor_matrix.py` | 7 skipped classes + rename refs | -| Verify | `tests/test_pipeline/test_serialization_helpers.py` | Confirm no skips remain | -| Verify | `tests/test_pipeline/test_integration_smoke.py` | Confirm no skips remain | - ---- - -## API Translation Reference - -Keep this table in mind throughout every migration task: - -| Old (`Pipeline`) | New (`PipelineJob`) | -|---|---| -| `Pipeline(name, pipeline_database=db)` | `PipelineJob(name, store=db)` | -| `pipeline.run(orchestrator=orch, observer=obs)` | `job.run(orchestrator=orch, observer=obs)` | -| `pipeline.label.get_all_records()` | `job.nodes["label"].get_all_records()` | -| `pipeline.dag` | `job.pipeline.dag` | -| `pipeline.show_graph()` | `job.pipeline.show_graph()` | -| `AsyncPipelineOrchestrator().run(pipeline.dag)` | `job.run(orchestrator=AsyncPipelineOrchestrator())` | -| `pipeline.compile(); pipeline.flush()` | Not needed — `with job:` compiles; `run()` flushes | - ---- - -## Task 1: Rename ExecutorType → OrchestratorType - -**Files:** -- Modify: `src/orcapod/types.py` -- Modify: `tests/test_channels/test_channels.py:547-568` -- Modify: `tests/test_pipeline/test_orchestrator_executor_matrix.py` (import line 46, usages at 88, 98) - -- [ ] **Step 1: Update `src/orcapod/types.py`** - - Replace the `ExecutorType` class and `PipelineConfig.executor` field. - - Find this block (lines ~274–309): - ```python - class ExecutorType(Enum): - """Pipeline execution strategy. - - Attributes: - SYNCHRONOUS: Current behavior -- ``static_process`` chain with - pull-based materialization. - ASYNC_CHANNELS: Push-based async channel execution via - ``async_execute``. - """ - - SYNCHRONOUS = "synchronous" - ASYNC_CHANNELS = "async_channels" - - - @dataclass(frozen=True, slots=True) - class PipelineConfig: - """Pipeline-level execution configuration. - - Attributes: - executor: Which execution strategy to use. - channel_buffer_size: Max items buffered per channel edge. - default_max_concurrency: Pipeline-wide default for per-node - concurrency. ``None`` means unlimited. - execution_engine: Optional data-function executor applied to all - function nodes (e.g. ``RayExecutor``). ``None`` means in-process - execution. - execution_engine_opts: Resource/options dict forwarded to the engine - via ``with_options()`` (e.g. ``{"num_cpus": 4}``). - """ - - executor: ExecutorType = ExecutorType.SYNCHRONOUS - channel_buffer_size: int = 64 - default_max_concurrency: int | None = None - execution_engine: DataFunctionExecutorProtocol | None = None - execution_engine_opts: dict[str, Any] | None = None - ``` - - Replace with: - ```python - class OrchestratorType(Enum): - """Pipeline orchestrator selection. - - Attributes: - SYNCHRONOUS: Pull-based synchronous DAG walk via - ``SyncPipelineOrchestrator``. - ASYNC_CHANNELS: Push-based async channel execution via - ``AsyncPipelineOrchestrator``. - - Note: - This is distinct from ``DataFunction.executor``, which controls - distributed execution of individual data functions (e.g. Ray). - ``OrchestratorType`` selects the pipeline-level coordination - strategy only. - """ - - SYNCHRONOUS = "synchronous" - ASYNC_CHANNELS = "async_channels" - - - @dataclass(frozen=True, slots=True) - class PipelineConfig: - """Pipeline-level execution configuration. - - Attributes: - orchestrator: Which orchestrator strategy to use. - channel_buffer_size: Max items buffered per channel edge. - default_max_concurrency: Pipeline-wide default for per-node - concurrency. ``None`` means unlimited. - execution_engine: Optional data-function executor applied to all - function nodes (e.g. ``RayExecutor``). ``None`` means in-process - execution. - execution_engine_opts: Resource/options dict forwarded to the engine - via ``with_options()`` (e.g. ``{"num_cpus": 4}``). - """ - - orchestrator: OrchestratorType = OrchestratorType.SYNCHRONOUS - channel_buffer_size: int = 64 - default_max_concurrency: int | None = None - execution_engine: DataFunctionExecutorProtocol | None = None - execution_engine_opts: dict[str, Any] | None = None - ``` - -- [ ] **Step 2: Update `tests/test_channels/test_channels.py`** - - Find lines ~547–568 and update four occurrences: - ```python - # Before (3 separate inline imports + usages): - from orcapod.types import ExecutorType - assert ExecutorType.SYNCHRONOUS.value == "synchronous" - assert ExecutorType.ASYNC_CHANNELS.value == "async_channels" - - from orcapod.types import ExecutorType, PipelineConfig - cfg = PipelineConfig() - assert cfg.executor == ExecutorType.SYNCHRONOUS - - from orcapod.types import ExecutorType, PipelineConfig - cfg = PipelineConfig( - executor=ExecutorType.ASYNC_CHANNELS, - ... - ) - assert cfg.executor == ExecutorType.ASYNC_CHANNELS - ``` - - Replace with: - ```python - from orcapod.types import OrchestratorType - assert OrchestratorType.SYNCHRONOUS.value == "synchronous" - assert OrchestratorType.ASYNC_CHANNELS.value == "async_channels" - - from orcapod.types import OrchestratorType, PipelineConfig - cfg = PipelineConfig() - assert cfg.orchestrator == OrchestratorType.SYNCHRONOUS - - from orcapod.types import OrchestratorType, PipelineConfig - cfg = PipelineConfig( - orchestrator=OrchestratorType.ASYNC_CHANNELS, - ... - ) - assert cfg.orchestrator == OrchestratorType.ASYNC_CHANNELS - ``` - -- [ ] **Step 3: Update `tests/test_pipeline/test_orchestrator_executor_matrix.py` import** - - Find line ~46: - ```python - from orcapod.types import ExecutorType, NodeConfig, PipelineConfig - ``` - Replace with: - ```python - from orcapod.types import NodeConfig, OrchestratorType, PipelineConfig - ``` - - Also find any remaining `ExecutorType.SYNCHRONOUS` / `ExecutorType.ASYNC_CHANNELS` references in this file (lines ~88, 98) and replace with `OrchestratorType.SYNCHRONOUS` / `OrchestratorType.ASYNC_CHANNELS`. - -- [ ] **Step 4: Verify no remaining `ExecutorType` references** - - ```bash - grep -rn "ExecutorType" src/ tests/ --include="*.py" - ``` - Expected: no output. - -- [ ] **Step 5: Run the full test suite to confirm no breakage** - - ```bash - uv run pytest tests/ -x -q 2>&1 | tail -20 - ``` - Expected: same pass/fail counts as before (skipped tests remain skipped). - -- [ ] **Step 6: Commit** - - ```bash - git add src/orcapod/types.py tests/test_channels/test_channels.py tests/test_pipeline/test_orchestrator_executor_matrix.py - git commit -m "refactor(types): rename ExecutorType to OrchestratorType - - ExecutorType was ambiguous — DataFunction also has an executor concept - (for distributed backends like Ray). OrchestratorType makes the - pipeline-level orchestration strategy explicit. Rename - PipelineConfig.executor → PipelineConfig.orchestrator accordingly. - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 2: Add `orchestrator=` Parameter to `PipelineJob.run()` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py:643-775` -- Modify: `tests/test_pipeline/test_pipeline_job.py` (add orchestrator param test) - -- [ ] **Step 1: Write a failing test in `tests/test_pipeline/test_pipeline_job.py`** - - Find the `TestPipelineJobRun` class and add this test at the end of it: - - ```python - def test_run_accepts_explicit_sync_orchestrator(self, tmp_path): - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - - db = InMemoryArrowDatabase() - source = _make_source([{"x": 1}, {"x": 2}]) - - @FunctionPod - def doubler(x: int) -> int: - return x * 2 - - job = PipelineJob(name="orch_param", store=db) - with job: - doubler(source, label="doubler") - - completed = job.run(orchestrator=SyncPipelineOrchestrator()) - records = completed.nodes["doubler"].get_all_records() - assert len(records) == 2 - - def test_run_accepts_async_orchestrator(self, tmp_path): - from orcapod.pipeline import AsyncPipelineOrchestrator - - db = InMemoryArrowDatabase() - source = _make_source([{"x": 1}, {"x": 2}]) - - @FunctionPod - def doubler(x: int) -> int: - return x * 2 - - job = PipelineJob(name="async_orch", store=db) - with job: - doubler(source, label="doubler") - - completed = job.run(orchestrator=AsyncPipelineOrchestrator()) - records = completed.nodes["doubler"].get_all_records() - assert len(records) == 2 - ``` - -- [ ] **Step 2: Run the new tests to confirm they fail** - - ```bash - uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobRun::test_run_accepts_explicit_sync_orchestrator tests/test_pipeline/test_pipeline_job.py::TestPipelineJobRun::test_run_accepts_async_orchestrator -v - ``` - Expected: `FAILED` — `run()` does not accept `orchestrator` keyword argument yet. - -- [ ] **Step 3: Modify `run()` in `src/orcapod/pipeline/job.py`** - - Find the `run()` method signature (line ~643): - ```python - def run( - self, - observer: "ExecutionObserverProtocol | None" = None, - ) -> "PipelineJob": - """Execute the resolvable subgraph of this job in place. - ... - Args: - observer: Optional execution observer. - ... - """ - import hashlib - import uuid - - from orcapod.pipeline.observer import NoOpObserver - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - ``` - - Replace with: - ```python - def run( - self, - orchestrator: "SyncPipelineOrchestrator | AsyncPipelineOrchestrator | None" = None, - observer: "ExecutionObserverProtocol | None" = None, - ) -> "PipelineJob": - """Execute the resolvable subgraph of this job in place. - ... - Args: - orchestrator: Orchestrator to use. Defaults to - ``SyncPipelineOrchestrator`` when ``None``. - observer: Optional execution observer. - ... - """ - import hashlib - import uuid - - from orcapod.pipeline.observer import NoOpObserver - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - ``` - - Then find the orchestrator call (line ~758): - ```python - SyncPipelineOrchestrator().run( - exec_dag, - observer=effective_observer, - run_id=run_id, - pipeline_uri=pipeline_uri, - ) - ``` - - Replace with: - ```python - effective_orchestrator = orchestrator or SyncPipelineOrchestrator() - effective_orchestrator.run( - exec_dag, - observer=effective_observer, - run_id=run_id, - pipeline_uri=pipeline_uri, - ) - ``` - - Also add `AsyncPipelineOrchestrator` to the `TYPE_CHECKING` imports at the top of the file. Find the `if TYPE_CHECKING:` block near the top of `job.py` and add: - ```python - if TYPE_CHECKING: - ... - from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - ``` - -- [ ] **Step 4: Run the new tests to confirm they pass** - - ```bash - uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobRun::test_run_accepts_explicit_sync_orchestrator tests/test_pipeline/test_pipeline_job.py::TestPipelineJobRun::test_run_accepts_async_orchestrator -v - ``` - Expected: both `PASSED`. - -- [ ] **Step 5: Run the full pipeline test suite to confirm no regressions** - - ```bash - uv run pytest tests/test_pipeline/ -x -q 2>&1 | tail -20 - ``` - Expected: same counts as before (skipped tests remain skipped, no new failures). - -- [ ] **Step 6: Commit** - - ```bash - git add src/orcapod/pipeline/job.py tests/test_pipeline/test_pipeline_job.py - git commit -m "feat(pipeline): add orchestrator= parameter to PipelineJob.run() - - Defaults to SyncPipelineOrchestrator when None, preserving existing - behavior. Allows async orchestrator tests to be migrated without - calling orchestrator internals directly. - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 3: Migrate `test_sync_orchestrator.py` - -**Files:** -- Migrate: `tests/test_pipeline/test_sync_orchestrator.py` - -This file has 8 skipped classes. The migration pattern for all of them is: -- Replace `Pipeline(name, pipeline_database=db)` → `PipelineJob(name, store=db)` -- Replace `orch.run(pipeline.dag)` / `pipeline.run(orchestrator=orch)` → `job.run()` or `job.run(orchestrator=orch)` -- Replace `pipeline.label.get_all_records()` → `job.nodes["label"].get_all_records()` -- Replace `result.node_outputs` access → `job.nodes["label"].get_all_records()` - -The `TestMaterializeResults` class tests `materialize_results=False` on the orchestrator. `PipelineJob.run()` exposes this parameter and passes it through to the orchestrator, so this class can be fully migrated. The migration pattern is the same as the other classes, plus the assertion must check `job.nodes["label"].get_all_records().num_rows` to confirm DB records are written regardless of the `materialize_results` setting. - -- [ ] **Step 1: Read the skipped test classes** - - ```bash - grep -n "class Test\|@pytest.mark.skip" tests/test_pipeline/test_sync_orchestrator.py - ``` - Expected output lists 8 `@pytest.mark.skip` decorators before class definitions. - -- [ ] **Step 2: Update imports at the top of the file** - - Find the old imports: - ```python - from orcapod.pipeline import Pipeline, SyncPipelineOrchestrator - ``` - Replace with (keep any existing `PipelineJob` import, add if missing): - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - ``` - -- [ ] **Step 3: Remove `@pytest.mark.skip` from all 8 classes** - - For each class, delete the line: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") - ``` - -- [ ] **Step 4: Update all fixtures and test methods to use `PipelineJob`** - - Apply these substitutions throughout every test method in the file: - - **Construction:** - ```python - # Before - pipeline = Pipeline(name="linear", pipeline_database=InMemoryArrowDatabase()) - with pipeline: - doubler(src, label="doubler") - - # After - job = PipelineJob(name="linear", store=InMemoryArrowDatabase()) - with job: - doubler(src, label="doubler") - ``` - - **Sync execution:** - ```python - # Before - orch = SyncPipelineOrchestrator() - orch.run(pipeline.dag) - - # After - job.run() # defaults to SyncPipelineOrchestrator - ``` - - **Async execution:** - ```python - # Before - AsyncPipelineOrchestrator().run(pipeline.dag) - - # After - job.run(orchestrator=AsyncPipelineOrchestrator()) - ``` - - **Node result access:** - ```python - # Before - records = pipeline.doubler.get_all_records() - - # After - records = job.nodes["doubler"].get_all_records() - ``` - - **Observer injection** (TestSyncOrchestratorObserver, TestSyncObserverInjection): - ```python - # Before - orch = SyncPipelineOrchestrator() - orch.run(pipeline.dag, observer=recording_observer) - - # After - job.run(observer=recording_observer) - ``` - - **TestSyncAsyncParity** (compares sync vs async results): - ```python - # Before - pipeline_sync = Pipeline(name="sync", pipeline_database=db_sync) - with pipeline_sync: - pod(src, label="node") - SyncPipelineOrchestrator().run(pipeline_sync.dag) - sync_records = pipeline_sync.node.get_all_records() - - pipeline_async = Pipeline(name="async", pipeline_database=db_async) - with pipeline_async: - pod(src, label="node") - AsyncPipelineOrchestrator().run(pipeline_async.dag) - async_records = pipeline_async.node.get_all_records() - - # After - job_sync = PipelineJob(name="sync", store=db_sync) - with job_sync: - pod(src, label="node") - job_sync.run() - sync_records = job_sync.nodes["node"].get_all_records() - - job_async = PipelineJob(name="async", store=db_async) - with job_async: - pod(src, label="node") - job_async.run(orchestrator=AsyncPipelineOrchestrator()) - async_records = job_async.nodes["node"].get_all_records() - ``` - -- [ ] **Step 5: Migrate `TestMaterializeResults`** - - Remove the `@pytest.mark.skip` decorator and migrate the class to use `PipelineJob`. Each test should call `job.run(materialize_results=False)` (or `True`) and verify that DB records are written regardless: - - ```python - class TestMaterializeResults: - def test_sync_materialize_false_db_records_persisted(self): - job = self._make_job() - job.run(materialize_results=False) - records = job.nodes["doubler"].get_all_records() - assert records is not None - assert records.num_rows == 3 - - def test_async_materialize_false_db_records_persisted(self): - job = self._make_job() - job.run(orchestrator=AsyncPipelineOrchestrator(), materialize_results=False) - records = job.nodes["doubler"].get_all_records() - assert records is not None - assert records.num_rows == 3 - ``` - -- [ ] **Step 6: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_sync_orchestrator.py -v 2>&1 | tail -30 - ``` - Expected: all previously-skipped tests now `PASSED`. Fix any failures before continuing. - -- [ ] **Step 7: Commit** - - ```bash - git add tests/test_pipeline/test_sync_orchestrator.py - git commit -m "test(sync_orchestrator): migrate skipped tests to PipelineJob API - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 4: Migrate `test_composite_observer.py` - -**Files:** -- Migrate: `tests/test_pipeline/test_composite_observer.py` - -4 skipped classes: `TestLoggingAndStatusTogether`, `TestCreateDataLoggerDelegation`, `TestContextualizeReturnsComposite`, `TestCompositeWithFailures`. - -All use `CompositeObserver(log_obs, status_obs)` passed to `pipeline.run()`. Migration: same PipelineJob substitution, pass observer to `job.run(observer=composite_obs)`. - -- [ ] **Step 1: Update imports** - - Find: - ```python - from orcapod.pipeline import Pipeline, SyncPipelineOrchestrator - ``` - Replace with: - ```python - from orcapod.pipeline import SyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - ``` - -- [ ] **Step 2: Remove `@pytest.mark.skip` from all 4 classes** - - Delete each line: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") - ``` - -- [ ] **Step 3: Update fixtures and test bodies** - - The shared fixture (if any) and each test method follow this pattern: - - ```python - # Before - pipeline = Pipeline(name="test_composite", pipeline_database=obs_db.at("test_composite")) - with pipeline: - pod(source, label="doubler") - - composite = CompositeObserver(log_obs, status_obs) - SyncPipelineOrchestrator().run(pipeline.dag, observer=composite) - - # After - job = PipelineJob(name="test_composite", store=obs_db.at("test_composite")) - with job: - pod(source, label="doubler") - - composite = CompositeObserver(log_obs, status_obs) - job.run(observer=composite) - ``` - - For `TestContextualizeReturnsComposite` which calls `observer.contextualize(...)` directly — no `run()` needed, just keep that test as-is (no Pipeline/PipelineJob involved). Remove the skip decorator only. - -- [ ] **Step 4: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_composite_observer.py -v 2>&1 | tail -20 - ``` - Expected: all 4 previously-skipped classes pass. Fix any failures. - -- [ ] **Step 5: Commit** - - ```bash - git add tests/test_pipeline/test_composite_observer.py - git commit -m "test(composite_observer): migrate skipped tests to PipelineJob API - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 5: Migrate `test_status_observer_integration.py` - -**Files:** -- Migrate: `tests/test_pipeline/test_status_observer_integration.py` - -11 skipped classes covering `StatusObserver` integration: success/failure status events, flat storage, queryable columns, async orchestrator, fail-fast policy, mixed success/failure, multi-node, run ID tracking, schema validation. - -- [ ] **Step 1: Update imports** - - Find: - ```python - from orcapod.pipeline import ( - AsyncPipelineOrchestrator, Pipeline, SyncPipelineOrchestrator, - ) - ``` - Replace with: - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - ``` - -- [ ] **Step 2: Remove `@pytest.mark.skip` from all 11 classes** - - Delete each: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") - ``` - -- [ ] **Step 3: Update fixtures and test bodies** - - **Sync orchestrator tests** (`TestSyncPipelineSuccessStatus`, `TestFailingDatasStatus`, `TestFlatStatusStorage`, `TestQueryableTagColumns`, `TestFailFastErrorPolicy`, `TestMixedSuccessFailure`, `TestMultipleFunctionNodesSeparateStatus`, `TestGetStatusNodeSpecific`, `TestStatusSchema`, `TestRunIdTracking`): - - ```python - # Before - pipeline = Pipeline(name="test_status", pipeline_database=db) - with pipeline: - pod(source, label="doubler") - obs = StatusObserver(status_database=db) - SyncPipelineOrchestrator().run(pipeline.dag, observer=obs) - status = obs.get_status() - - # After - job = PipelineJob(name="test_status", store=db) - with job: - pod(source, label="doubler") - obs = StatusObserver(status_database=db) - job.run(observer=obs) - status = obs.get_status() - ``` - - **Async orchestrator tests** (`TestAsyncOrchestratorStatus`): - - ```python - # Before - AsyncPipelineOrchestrator().run(pipeline.dag, observer=obs) - - # After - job.run(orchestrator=AsyncPipelineOrchestrator(), observer=obs) - ``` - - **Node result access where used:** - ```python - # Before - records = pipeline.doubler.get_all_records() - # After - records = job.nodes["doubler"].get_all_records() - ``` - -- [ ] **Step 4: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_status_observer_integration.py -v 2>&1 | tail -30 - ``` - Expected: all 11 previously-skipped classes pass. Fix any failures. - -- [ ] **Step 5: Commit** - - ```bash - git add tests/test_pipeline/test_status_observer_integration.py - git commit -m "test(status_observer): migrate skipped tests to PipelineJob API - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 6: Migrate `test_logging_observer_integration.py` - -**Files:** -- Migrate: `tests/test_pipeline/test_logging_observer_integration.py` - -9 skipped classes covering `LoggingObserver` integration: stdout/stderr capture, traceback on failure, flat log storage, queryable tag columns, async orchestrator logs, fail-fast, mixed success/failure, multi-node combined logs, per-node log query. - -- [ ] **Step 1: Update imports** - - Find: - ```python - from orcapod.pipeline import ( - AsyncPipelineOrchestrator, Pipeline, SyncPipelineOrchestrator, - ) - ``` - Replace with: - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - ``` - -- [ ] **Step 2: Remove `@pytest.mark.skip` from all 9 classes** - - Delete each: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") - ``` - -- [ ] **Step 3: Update fixtures and test bodies** - - **Sync orchestrator tests** (`TestSyncPipelineSuccessLogs`, `TestFailingDatasLogged`, `TestFlatLogStorage`, `TestQueryableTagColumns`, `TestFailFastErrorPolicy`, `TestMixedSuccessFailure`, `TestMultipleFunctionNodesCombinedLogs`, `TestGetLogsNodeSpecific`): - - ```python - # Before - pipeline = Pipeline(name="test_logs", pipeline_database=db) - with pipeline: - pod(source, label="doubler") - obs = LoggingObserver(log_database=db) - SyncPipelineOrchestrator().run(pipeline.dag, observer=obs) - logs = obs.get_logs() - - # After - job = PipelineJob(name="test_logs", store=db) - with job: - pod(source, label="doubler") - obs = LoggingObserver(log_database=db) - job.run(observer=obs) - logs = obs.get_logs() - ``` - - **Async orchestrator test** (`TestAsyncOrchestratorLogs`): - - ```python - # Before - AsyncPipelineOrchestrator().run(pipeline.dag, observer=obs) - - # After - job.run(orchestrator=AsyncPipelineOrchestrator(), observer=obs) - ``` - -- [ ] **Step 4: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_logging_observer_integration.py -v 2>&1 | tail -30 - ``` - Expected: all 9 previously-skipped classes pass. Fix any failures. - -- [ ] **Step 5: Commit** - - ```bash - git add tests/test_pipeline/test_logging_observer_integration.py - git commit -m "test(logging_observer): migrate skipped tests to PipelineJob API - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 7: Migrate `test_orchestrator.py` (Async Orchestrator) - -**Files:** -- Migrate: `tests/test_pipeline/test_orchestrator.py` - -8 skipped classes testing `AsyncPipelineOrchestrator`: linear, operator, diamond DAG, run_async from event loop, buffer size, fan-out, error propagation, observer injection. - -The special case is `TestOrchestratorRunAsync` which used `await orchestrator.run_async(pipeline.dag)`. In the new API, `job.run(orchestrator=AsyncPipelineOrchestrator())` calls the sync entry point which internally drives the event loop. Migrate `TestOrchestratorRunAsync` to use `job.run(orchestrator=AsyncPipelineOrchestrator())` — the async execution path is still exercised. If the test specifically needs to call `run_async()` from within a running event loop, retain the `@pytest.mark.asyncio` decorator and call `await AsyncPipelineOrchestrator().run_async(exec_dag)` after building the dag via `job.pipeline.dag` as a fallback — but prefer `job.run()` first. - -- [ ] **Step 1: Update imports** - - Find: - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator, Pipeline - ``` - Replace with: - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - ``` - -- [ ] **Step 2: Remove `@pytest.mark.skip` from all 8 classes** - - Delete each: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") - ``` - -- [ ] **Step 3: Update fixtures and test bodies** - - **Standard async orchestrator tests** (`TestOrchestratorLinearPipeline`, `TestOrchestratorOperatorPipeline`, `TestOrchestratorDiamondDag`, `TestBufferSizeConfiguration`, `TestAsyncOrchestratorFanOut`, `TestAsyncOrchestratorErrorPropagation`, `TestAsyncOrchestratorObserverInjection`): - - ```python - # Before - pipeline = Pipeline(name="linear", pipeline_database=InMemoryArrowDatabase()) - with pipeline: - pod(src, label="doubler") - pipeline.compile() - AsyncPipelineOrchestrator().run(pipeline.dag) - pipeline.flush() - records = pipeline.doubler.get_all_records() - - # After - job = PipelineJob(name="linear", store=InMemoryArrowDatabase()) - with job: - pod(src, label="doubler") - job.run(orchestrator=AsyncPipelineOrchestrator()) - records = job.nodes["doubler"].get_all_records() - ``` - - **With custom buffer size:** - ```python - # Before - AsyncPipelineOrchestrator(buffer_size=4).run(pipeline.dag) - - # After - job.run(orchestrator=AsyncPipelineOrchestrator(buffer_size=4)) - ``` - - **With observer:** - ```python - # Before - AsyncPipelineOrchestrator().run(pipeline.dag, observer=recording_obs) - - # After - job.run(orchestrator=AsyncPipelineOrchestrator(), observer=recording_obs) - ``` - - **`TestOrchestratorRunAsync`** — keep `@pytest.mark.asyncio`, migrate to use `job.run()`: - ```python - # Before - async def test_run_async_from_event_loop(self): - pipeline = Pipeline(name="async_loop", pipeline_database=db) - with pipeline: - pod(src, label="doubler") - pipeline.compile() - await AsyncPipelineOrchestrator().run_async(pipeline.dag) - pipeline.flush() - - # After - async def test_run_async_from_event_loop(self): - # PipelineJob.run() drives the event loop synchronously. - # Test that async execution produces correct results. - job = PipelineJob(name="async_loop", store=db) - with job: - pod(src, label="doubler") - job.run(orchestrator=AsyncPipelineOrchestrator()) - records = job.nodes["doubler"].get_all_records() - assert len(records) > 0 - ``` - -- [ ] **Step 4: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_orchestrator.py -v 2>&1 | tail -30 - ``` - Expected: all 8 previously-skipped classes pass. Fix any failures. - -- [ ] **Step 5: Commit** - - ```bash - git add tests/test_pipeline/test_orchestrator.py - git commit -m "test(orchestrator): migrate async orchestrator tests to PipelineJob API - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 8: Migrate `test_graph_rendering.py` - -**Files:** -- Migrate: `tests/test_pipeline/test_graph_rendering.py` - -8 skipped classes testing `GraphRenderer`, node attributes, HTML labels, convenience functions, `Pipeline.show_graph()`, full render paths. - -Key pattern difference: the old tests used `compiled_pipeline.dag` (a `Pipeline` fixture). In the new API, `job.pipeline.dag` is available after the `with` block exits — no `run()` call needed. - -- [ ] **Step 1: Update imports** - - Find: - ```python - from orcapod.pipeline import Pipeline - ``` - Replace with: - ```python - from orcapod.pipeline.job import PipelineJob - ``` - -- [ ] **Step 2: Remove `@pytest.mark.skip` from all 8 classes** - - Delete each: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") - ``` - -- [ ] **Step 3: Update shared fixtures** - - Find the `compiled_pipeline` and `node_graph` fixtures. They likely look like: - ```python - @pytest.fixture - def compiled_pipeline(pipeline_db): - pipeline = Pipeline(name="render_test", pipeline_database=pipeline_db) - with pipeline: - pod(source, label="node") - return pipeline - - @pytest.fixture - def node_graph(compiled_pipeline): - return compiled_pipeline.dag - ``` - - Replace with: - ```python - @pytest.fixture - def compiled_job(pipeline_db): - job = PipelineJob(name="render_test", store=pipeline_db) - with job: - pod(source, label="node") - return job - - @pytest.fixture - def node_graph(compiled_job): - return compiled_job.pipeline.dag - ``` - - Update all test methods that accept `compiled_pipeline` as a parameter to accept `compiled_job` instead, and replace `compiled_pipeline.dag` → `compiled_job.pipeline.dag` and `compiled_pipeline.show_graph()` → `compiled_job.pipeline.show_graph()`. - -- [ ] **Step 4: Update `TestPipelineShowGraph` tests** - - ```python - # Before - def test_show_graph_raises_before_compile(self): - pipeline = Pipeline(name="no_compile", pipeline_database=db) - with pytest.raises(RuntimeError): - pipeline.show_graph() - - def test_show_graph_returns_dot(self, compiled_pipeline): - dot = compiled_pipeline.show_graph() - assert "digraph" in dot - - # After - def test_show_graph_raises_before_compile(self): - job = PipelineJob(name="no_compile", store=db) - with pytest.raises(RuntimeError): - job.pipeline.show_graph() # raises because job not compiled yet - - def test_show_graph_returns_dot(self, compiled_job): - dot = compiled_job.pipeline.show_graph() - assert "digraph" in dot - ``` - -- [ ] **Step 5: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_graph_rendering.py -v 2>&1 | tail -30 - ``` - Expected: all 8 previously-skipped classes pass. Fix any failures. - -- [ ] **Step 6: Commit** - - ```bash - git add tests/test_pipeline/test_graph_rendering.py - git commit -m "test(graph_rendering): migrate skipped tests to PipelineJob API - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 9: Migrate `test_orchestrator_executor_matrix.py` - -**Files:** -- Migrate: `tests/test_pipeline/test_orchestrator_executor_matrix.py` - -7 skipped classes testing all combinations of orchestrator × function type. The two already-passing classes (`TestSyncOrchestratorSyncFunctionPipelineJob`) use PipelineJob already — keep them as-is. Also update any remaining `OrchestratorType` / `PipelineConfig.orchestrator` references from Task 1 (should already be done). - -- [ ] **Step 1: Confirm `ExecutorType` rename is complete in this file** - - ```bash - grep -n "ExecutorType" tests/test_pipeline/test_orchestrator_executor_matrix.py - ``` - Expected: no output (renamed in Task 1). - -- [ ] **Step 2: Remove `@pytest.mark.skip` from all 7 classes** - - Delete each: - ```python - @pytest.mark.skip(reason="Migrating to PipelineJob-based API — see ENG-491") - ``` - -- [ ] **Step 3: Update imports** - - Find any remaining `Pipeline` imports: - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator, Pipeline - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - ``` - Replace with: - ```python - from orcapod.pipeline import AsyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - ``` - -- [ ] **Step 4: Replace all `PipelineConfig`-based dispatch with direct orchestrator instances** - - The matrix tests used `PipelineConfig(executor=...)` to select orchestrators. Replace all occurrences: - - ```python - # Before (sync orchestrator cell) - pipeline = Pipeline(name="matrix", pipeline_database=db, auto_compile=True) - with pipeline: - pod(source, label="node") - pipeline.run(config=PipelineConfig(orchestrator=OrchestratorType.SYNCHRONOUS)) - records = pipeline.node.get_all_records() - - # After - job = PipelineJob(name="matrix", store=db) - with job: - pod(source, label="node") - job.run(orchestrator=SyncPipelineOrchestrator()) - records = job.nodes["node"].get_all_records() - ``` - - ```python - # Before (async orchestrator cell) - pipeline.run(config=PipelineConfig(orchestrator=OrchestratorType.ASYNC_CHANNELS)) - - # After - job.run(orchestrator=AsyncPipelineOrchestrator()) - ``` - - For the concurrency/performance tests (`TestConcurrencyBenefitAcrossMatrix`, `TestAsyncAsyncVsAsyncSync`), update timing assertions if needed — `job.run(orchestrator=AsyncPipelineOrchestrator())` replaces the async channel path. - -- [ ] **Step 5: Remove `PipelineConfig` import if no longer used** - - ```bash - grep -n "PipelineConfig" tests/test_pipeline/test_orchestrator_executor_matrix.py - ``` - If no usages remain, remove it from the import line. - -- [ ] **Step 6: Run the migrated tests** - - ```bash - uv run pytest tests/test_pipeline/test_orchestrator_executor_matrix.py -v 2>&1 | tail -30 - ``` - Expected: all 7 previously-skipped classes pass. Fix any failures. - -- [ ] **Step 7: Commit** - - ```bash - git add tests/test_pipeline/test_orchestrator_executor_matrix.py - git commit -m "test(executor_matrix): migrate skipped tests to PipelineJob API - - Replace PipelineConfig-based orchestrator dispatch with direct orchestrator - instances. All four execution matrix cells now use PipelineJob.run(). - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - ---- - -## Task 10: Final Verification and Skip-Reason Audit - -**Files:** -- Verify: `tests/test_pipeline/test_serialization_helpers.py` -- Verify: `tests/test_pipeline/test_integration_smoke.py` -- Audit: all `tests/test_pipeline/` files - -- [ ] **Step 1: Confirm Group E files have no skipped tests** - - ```bash - grep -n "Migrating to PipelineJob-based API" \ - tests/test_pipeline/test_serialization_helpers.py \ - tests/test_pipeline/test_integration_smoke.py - ``` - Expected: no output (pre-implementation exploration confirmed these files have no remaining migration skips). If any skips appear, apply the same PipelineJob migration pattern used in Tasks 3–9. - -- [ ] **Step 2: Audit all pipeline test files for remaining skip reasons** - - ```bash - grep -rn "Migrating to PipelineJob-based API" tests/test_pipeline/ - ``` - Expected: no output. All migration skip reasons must be gone. - -- [ ] **Step 3: Run the full pipeline test suite** - - ```bash - uv run pytest tests/test_pipeline/ -v 2>&1 | tail -40 - ``` - Expected: all previously-skipped tests now pass. Zero tests with the migration skip reason. - -- [ ] **Step 4: Run the full test suite** - - ```bash - uv run pytest tests/ -q 2>&1 | tail -20 - ``` - Expected: full suite passes. Note total pass count. - -- [ ] **Step 5: Commit if any Group E changes were needed** - - ```bash - git add tests/test_pipeline/test_serialization_helpers.py tests/test_pipeline/test_integration_smoke.py - git commit -m "test(pipeline): confirm Group E files fully migrated - - Co-Authored-By: Claude Sonnet 4.6 " - ``` - (Only commit if files were actually changed.) - -- [ ] **Step 6: Push the branch** - - ```bash - git push -u origin eywalker/eng-491-migrate-remaining-skipped-pipeline-tests-to-pipelinejob-api - ``` diff --git a/superpowers/plans/2026-06-05-eng-516-source-node-column-config-forwarding.md b/superpowers/plans/2026-06-05-eng-516-source-node-column-config-forwarding.md deleted file mode 100644 index c27d6531a..000000000 --- a/superpowers/plans/2026-06-05-eng-516-source-node-column-config-forwarding.md +++ /dev/null @@ -1,825 +0,0 @@ -# ENG-516: SourceNodeBase column_config / all_info forwarding — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix `SourceNodeBase.keys()` and `output_schema()` to honour `column_config` / `all_info` — computing system-tag column names for unbound nodes and forwarding the arguments to `bound_source` for bound `SourceJobNode` instances. - -**Architecture:** Extract two shared utility functions (`compute_schema_hash` in `schema_utils.py`, `system_tag_column_names` in `arrow_utils.py`), use them from both `stream_builder.py` (refactor) and `SourceNodeBase` (new behaviour). Then add `keys()` / `output_schema()` overrides to `SourceJobNode` that delegate to `bound_source` when set. - -**Tech Stack:** Python, PyArrow, orcapod internal types (`Schema`, `ColumnConfig`), `uv run pytest`. - -**Spec:** `superpowers/specs/2026-06-05-eng-516-source-node-column-config-forwarding.md` - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/utils/schema_utils.py` | Modify | Add `compute_schema_hash()` | -| `src/orcapod/utils/arrow_utils.py` | Modify | Add `system_tag_column_names()`; use it inside `add_system_tag_columns()` | -| `src/orcapod/core/sources/stream_builder.py` | Modify | Call `compute_schema_hash()` instead of inlining hash logic | -| `src/orcapod/core/nodes/source_node.py` | Modify | Add `_schema_hash()` helper; fix `SourceNodeBase.keys()` and `output_schema()`; add `SourceJobNode.keys()` and `output_schema()` overrides | -| `tests/test_utils/test_schema_utils.py` | Create | Unit tests for `compute_schema_hash()` | -| `tests/test_utils/test_arrow_utils.py` | Modify | Add tests for `system_tag_column_names()` | -| `tests/test_core/nodes/test_source_node.py` | Modify | Add `TestSourceNodeColumnConfig` class | - ---- - -## Task 1: Add `compute_schema_hash()` to `schema_utils.py` - -**Files:** -- Create: `tests/test_utils/test_schema_utils.py` -- Modify: `src/orcapod/utils/schema_utils.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_utils/test_schema_utils.py`: - -```python -"""Tests for schema_utils utility functions.""" - -from __future__ import annotations - -import pytest - -from orcapod.config import DEFAULT_CONFIG -from orcapod.contexts import resolve_context -from orcapod.types import Schema - - -@pytest.fixture -def semantic_hasher(): - return resolve_context(None).semantic_hasher - - -@pytest.fixture -def char_count(): - return DEFAULT_CONFIG.schema_hash_n_char - - -class TestComputeSchemaHash: - def test_returns_string(self, semantic_hasher, char_count): - from orcapod.utils.schema_utils import compute_schema_hash - - result = compute_schema_hash( - Schema({"id": int}), - Schema({"value": float}), - semantic_hasher, - char_count, - ) - assert isinstance(result, str) - - def test_deterministic(self, semantic_hasher, char_count): - from orcapod.utils.schema_utils import compute_schema_hash - - tag = Schema({"id": int}) - data = Schema({"value": float}) - assert compute_schema_hash(tag, data, semantic_hasher, char_count) == \ - compute_schema_hash(tag, data, semantic_hasher, char_count) - - def test_length_equals_char_count(self, semantic_hasher, char_count): - from orcapod.utils.schema_utils import compute_schema_hash - - result = compute_schema_hash( - Schema({"id": int}), Schema({"value": float}), semantic_hasher, char_count - ) - assert len(result) == char_count - - def test_different_schemas_produce_different_hashes(self, semantic_hasher, char_count): - from orcapod.utils.schema_utils import compute_schema_hash - - h1 = compute_schema_hash(Schema({"id": int}), Schema({"v": float}), semantic_hasher, char_count) - h2 = compute_schema_hash(Schema({"id": int}), Schema({"v": int}), semantic_hasher, char_count) - assert h1 != h2 -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_utils/test_schema_utils.py -v -``` - -Expected: `ImportError` — `cannot import name 'compute_schema_hash' from 'orcapod.utils.schema_utils'` - -- [ ] **Step 3: Add `compute_schema_hash()` to `schema_utils.py`** - -Open `src/orcapod/utils/schema_utils.py`. After the existing imports block (after `from orcapod.types import Schema, SchemaLike`), add `Any` to the `typing` import if not already present (it is — line 7). Then append the new function at the end of the file: - -```python -def compute_schema_hash( - tag_schema: Schema, - data_schema: Schema, - semantic_hasher: Any, - char_count: int, -) -> str: - """Compute the schema hash used for system-tag column naming. - - This is the same hash that ``SourceStreamBuilder`` embeds in system-tag - column names when building an ``ArrowTableStream`` from a source table. - Extracting it here lets ``SourceNodeBase`` predict the system-tag column - names from its declared schemas alone, without requiring a live source. - - Args: - tag_schema: Python tag schema (``Schema`` mapping column names to types). - data_schema: Python data schema. - semantic_hasher: Hasher from the active ``DataContext`` - (``data_context.semantic_hasher``). - char_count: Number of hex characters to use in the output string - (``OrcapodConfig.schema_hash_n_char``). - - Returns: - Hex string of length ``char_count``. - """ - return semantic_hasher.hash_object( - (tag_schema, data_schema) - ).to_hex(char_count=char_count) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -uv run pytest tests/test_utils/test_schema_utils.py -v -``` - -Expected: All 4 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_utils/test_schema_utils.py src/orcapod/utils/schema_utils.py -git commit -m "feat(schema_utils): add compute_schema_hash() utility" -``` - ---- - -## Task 2: Add `system_tag_column_names()` to `arrow_utils.py` - -**Files:** -- Modify: `src/orcapod/utils/arrow_utils.py` -- Modify: `tests/test_utils/test_arrow_utils.py` - -- [ ] **Step 1: Write the failing tests** - -Open `tests/test_utils/test_arrow_utils.py` and append a new test class at the end: - -```python -class TestSystemTagColumnNames: - def test_returns_two_strings(self): - from orcapod.utils.arrow_utils import system_tag_column_names - - src_col, rec_col = system_tag_column_names("abc123") - assert isinstance(src_col, str) - assert isinstance(rec_col, str) - - def test_source_id_col_starts_with_system_tag_prefix(self): - from orcapod.system_constants import constants - from orcapod.utils.arrow_utils import system_tag_column_names - - src_col, _ = system_tag_column_names("abc123") - assert src_col.startswith(constants.SYSTEM_TAG_PREFIX) - - def test_record_id_col_starts_with_system_tag_prefix(self): - from orcapod.system_constants import constants - from orcapod.utils.arrow_utils import system_tag_column_names - - _, rec_col = system_tag_column_names("abc123") - assert rec_col.startswith(constants.SYSTEM_TAG_PREFIX) - - def test_schema_hash_embedded_in_col_names(self): - from orcapod.utils.arrow_utils import system_tag_column_names - - schema_hash = "deadbeef" - src_col, rec_col = system_tag_column_names(schema_hash) - assert schema_hash in src_col - assert schema_hash in rec_col - - def test_different_hashes_produce_different_names(self): - from orcapod.utils.arrow_utils import system_tag_column_names - - src1, rec1 = system_tag_column_names("aaa") - src2, rec2 = system_tag_column_names("bbb") - assert src1 != src2 - assert rec1 != rec2 - - def test_col_names_match_add_system_tag_columns_output(self): - """Column names from system_tag_column_names() must match those - added to a table by add_system_tag_columns().""" - import pyarrow as pa - from orcapod.utils.arrow_utils import add_system_tag_columns, system_tag_column_names - - schema_hash = "testhash" - table = pa.table({"id": pa.array([1]), "v": pa.array([1.0])}) - enriched = add_system_tag_columns(table, schema_hash, ["src_a"], ["row_0"]) - src_col, rec_col = system_tag_column_names(schema_hash) - assert src_col in enriched.column_names - assert rec_col in enriched.column_names -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_utils/test_arrow_utils.py::TestSystemTagColumnNames -v -``` - -Expected: `ImportError` — `cannot import name 'system_tag_column_names'` - -- [ ] **Step 3: Add `system_tag_column_names()` and refactor `add_system_tag_columns()`** - -Open `src/orcapod/utils/arrow_utils.py`. Find `add_system_tag_columns` (around line 940). Insert the new function directly above it, then update `add_system_tag_columns` to call it: - -```python -def system_tag_column_names(schema_hash: str) -> tuple[str, str]: - """Return the (source_id_col_name, record_id_col_name) system-tag column names. - - These are the two column names that ``add_system_tag_columns`` adds to every - source table, and that ``ArrowTableStream`` exposes via ``keys(system_tags=True)`` - and ``output_schema(system_tags=True)``. - - Args: - schema_hash: Hex schema hash produced by ``compute_schema_hash()``. - - Returns: - Tuple of ``(source_id_column_name, record_id_column_name)``. - Both start with ``constants.SYSTEM_TAG_PREFIX`` and have ``large_string`` - type in the Arrow table. - """ - source_id_col = ( - f"{constants.SYSTEM_TAG_SOURCE_ID_PREFIX}{constants.BLOCK_SEPARATOR}{schema_hash}" - ) - record_id_col = ( - f"{constants.SYSTEM_TAG_RECORD_ID_PREFIX}{constants.BLOCK_SEPARATOR}{schema_hash}" - ) - return source_id_col, record_id_col - - -def add_system_tag_columns( - table: "pa.Table", - schema_hash: str, - source_ids: str | Collection[str], - record_ids: Collection[str], -) -> "pa.Table": - """Add paired source_id and record_id system tag columns to an Arrow table.""" - if not table.column_names: - raise ValueError("Table is empty") - - # Normalize source_ids - if isinstance(source_ids, str): - source_ids = [source_ids] * table.num_rows - else: - source_ids = list(source_ids) - if len(source_ids) != table.num_rows: - raise ValueError( - "Length of source_ids must match number of rows in the table." - ) - - record_ids = list(record_ids) - if len(record_ids) != table.num_rows: - raise ValueError("Length of record_ids must match number of rows in the table.") - - source_id_col_name, record_id_col_name = system_tag_column_names(schema_hash) - - source_id_array = pa.array(source_ids, type=pa.large_string()) - record_id_array = pa.array(record_ids, type=pa.large_string()) - - # System tag columns are always computed, never null — declare nullable=False - # explicitly so the schema intent is not lost in Polars round-trips. - table = table.append_column( - pa.field(source_id_col_name, pa.large_string(), nullable=False), source_id_array - ) - table = table.append_column( - pa.field(record_id_col_name, pa.large_string(), nullable=False), record_id_array - ) - return table -``` - -- [ ] **Step 4: Run all tests to verify nothing broke** - -```bash -uv run pytest tests/test_utils/test_arrow_utils.py -v -``` - -Expected: All tests including the new `TestSystemTagColumnNames` class PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/utils/arrow_utils.py tests/test_utils/test_arrow_utils.py -git commit -m "feat(arrow_utils): add system_tag_column_names(); use it in add_system_tag_columns()" -``` - ---- - -## Task 3: Refactor `stream_builder.py` to use `compute_schema_hash()` - -**Files:** -- Modify: `src/orcapod/core/sources/stream_builder.py` - -This is a pure refactor — no new tests. The existing `test_stream_builder.py` is the regression suite. - -- [ ] **Step 1: Run existing stream_builder tests to establish baseline** - -```bash -uv run pytest tests/test_core/sources/test_stream_builder.py -v -``` - -Expected: All tests PASS. - -- [ ] **Step 2: Add import and replace inline hash computation** - -Open `src/orcapod/core/sources/stream_builder.py`. - -Add the import after the existing `from orcapod.types import ContentHash` line: - -```python -from orcapod.utils.schema_utils import compute_schema_hash -``` - -Find the schema hash computation block inside `SourceStreamBuilder.build()` (currently around lines 117–131): - -```python - # 4. Compute schema hash from tag/data python schemas. - # Nullable flags in the incoming table are trusted as-is — callers must - # set them correctly before calling build(). - non_sys = arrow_utils.drop_system_columns(table) - tag_schema = non_sys.select(list(tag_columns_tuple)).schema - data_schema = non_sys.drop(list(tag_columns_tuple)).schema - tag_python = self._data_context.type_converter.arrow_schema_to_python_schema( - tag_schema - ) - data_python = self._data_context.type_converter.arrow_schema_to_python_schema( - data_schema - ) - schema_hash = self._data_context.semantic_hasher.hash_object( - (tag_python, data_python) - ).to_hex(char_count=self._config.schema_hash_n_char) -``` - -Replace it with: - -```python - # 4. Compute schema hash from tag/data python schemas. - # Nullable flags in the incoming table are trusted as-is — callers must - # set them correctly before calling build(). - non_sys = arrow_utils.drop_system_columns(table) - tag_schema = non_sys.select(list(tag_columns_tuple)).schema - data_schema = non_sys.drop(list(tag_columns_tuple)).schema - tag_python = self._data_context.type_converter.arrow_schema_to_python_schema( - tag_schema - ) - data_python = self._data_context.type_converter.arrow_schema_to_python_schema( - data_schema - ) - schema_hash = compute_schema_hash( - tag_python, - data_python, - self._data_context.semantic_hasher, - self._config.schema_hash_n_char, - ) -``` - -- [ ] **Step 3: Run the tests to verify nothing broke** - -```bash -uv run pytest tests/test_core/sources/test_stream_builder.py -v -``` - -Expected: All tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/core/sources/stream_builder.py -git commit -m "refactor(stream_builder): use compute_schema_hash() from schema_utils" -``` - ---- - -## Task 4: Fix `SourceNodeBase` — honour `system_tags` in `keys()` and `output_schema()` - -**Files:** -- Modify: `tests/test_core/nodes/test_source_node.py` -- Modify: `src/orcapod/core/nodes/source_node.py` - -- [ ] **Step 1: Write the failing tests** - -Open `tests/test_core/nodes/test_source_node.py`. Append a new test class after the existing `TestSourceNodeAsTable` class: - -```python -class TestSourceNodeColumnConfig: - """SourceNodeBase must honour column_config / all_info in keys() and output_schema().""" - - def test_unbound_keys_default_unchanged(self, tag_schema, data_schema): - """Regression: keys() with no args still returns plain schema keys.""" - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - tag_keys, data_keys = node.keys() - assert tag_keys == ("id",) - assert data_keys == ("value",) - - def test_unbound_keys_system_tags_includes_both_system_cols(self, tag_schema, data_schema): - """keys(system_tags=True) adds both system-tag column names to tag_keys.""" - from orcapod.core.nodes.source_node import SourceNode - from orcapod.system_constants import constants - from orcapod.types import ColumnConfig - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - tag_keys, data_keys = node.keys(columns=ColumnConfig(system_tags=True)) - - assert data_keys == ("value",) - assert len(tag_keys) == 3 # id + source_id_col + record_id_col - assert tag_keys[0] == "id" - # Both extra columns start with the system-tag prefix - assert all(k.startswith(constants.SYSTEM_TAG_PREFIX) for k in tag_keys[1:]) - - def test_unbound_keys_all_info_same_as_system_tags(self, tag_schema, data_schema): - """keys(all_info=True) produces the same result as keys(system_tags=True).""" - from orcapod.core.nodes.source_node import SourceNode - from orcapod.types import ColumnConfig - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - via_system_tags = node.keys(columns=ColumnConfig(system_tags=True)) - via_all_info = node.keys(all_info=True) - assert via_system_tags == via_all_info - - def test_unbound_output_schema_system_tags_adds_str_entries(self, tag_schema, data_schema): - """output_schema(system_tags=True) tag schema includes two str-typed system-tag entries.""" - from orcapod.core.nodes.source_node import SourceNode - from orcapod.system_constants import constants - from orcapod.types import ColumnConfig - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - extended_tag_schema, data_schema_out = node.output_schema( - columns=ColumnConfig(system_tags=True) - ) - - assert data_schema_out == data_schema - assert "id" in extended_tag_schema - system_tag_entries = { - k: v for k, v in extended_tag_schema.items() - if k.startswith(constants.SYSTEM_TAG_PREFIX) - } - assert len(system_tag_entries) == 2 - assert all(v is str for v in system_tag_entries.values()) - - def test_unbound_output_schema_default_unchanged(self, tag_schema, data_schema): - """Regression: output_schema() with no args still returns plain schemas.""" - from orcapod.core.nodes.source_node import SourceNode - - node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - t, d = node.output_schema() - assert t == tag_schema - assert d == data_schema -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py::TestSourceNodeColumnConfig -v -``` - -Expected: `test_unbound_keys_default_unchanged` PASSES (existing behaviour). All `system_tags` tests FAIL with wrong key counts or missing columns. - -- [ ] **Step 3: Add imports to `source_node.py`** - -Open `src/orcapod/core/nodes/source_node.py`. Find the existing imports block. Add these two lines after `from orcapod.types import ColumnConfig, Schema`: - -```python -from orcapod.utils.arrow_utils import system_tag_column_names -from orcapod.utils.schema_utils import compute_schema_hash -``` - -- [ ] **Step 4: Add `_schema_hash()` helper to `SourceNodeBase`** - -Inside `SourceNodeBase`, find the `# Properties` section (after line ~84). Add this private helper after the `data_schema` property: - -```python -def _schema_hash(self) -> str: - """Compute the schema hash used for system-tag column naming. - - Produces the same hash that ``SourceStreamBuilder`` embeds in system-tag - column names, making it possible to predict those names from the declared - schemas alone without requiring a live source. - - Returns: - Hex string schema hash. - """ - return compute_schema_hash( - self._tag_schema, - self._data_schema, - self.data_context.semantic_hasher, - self.orcapod_config.schema_hash_n_char, - ) -``` - -- [ ] **Step 5: Replace `SourceNodeBase.keys()` body** - -Find `SourceNodeBase.keys()` (currently around line 159). Replace its body and docstring with: - -```python - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Return ``(tag_keys, data_keys)``. - - When ``columns.system_tags`` is ``True`` (or ``all_info=True``), the two - system-tag columns for this node's schema are appended to ``tag_keys``. - Their names are deterministic from the declared schemas and match what a - concrete source with the same schema would produce. - - Other ``ColumnConfig`` flags (``meta``, ``context``, ``source``, - ``content_hash``, ``sort_by_tags``) are no-ops at the node level — - consistent with ``ArrowTableStream.keys()`` which also ignores them. - - Args: - columns: Column selection config. - all_info: If ``True``, equivalent to ``ColumnConfig(system_tags=True)`` - for this method. - - Returns: - Tuple of ``(tag_column_names, data_column_names)``. - """ - columns_config = ColumnConfig.handle_config(columns, all_info=all_info) - tag_keys = tuple(self._tag_schema.keys()) - if columns_config.system_tags: - tag_keys += system_tag_column_names(self._schema_hash()) - return tag_keys, tuple(self._data_schema.keys()) -``` - -- [ ] **Step 6: Replace `SourceNodeBase.output_schema()` body** - -Find `SourceNodeBase.output_schema()` (currently around line 142). Replace its body and docstring with: - -```python - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return ``(tag_schema, data_schema)``. - - When ``columns.system_tags`` is ``True`` (or ``all_info=True``), the - returned tag schema is extended with the two system-tag entries for this - node's schema (both typed as ``str``). Their names are deterministic from - the declared schemas and match what a concrete source with the same schema - would produce. - - Other ``ColumnConfig`` flags (``meta``, ``context``, ``source``, - ``content_hash``, ``sort_by_tags``) are no-ops at the node level — - consistent with ``ArrowTableStream.output_schema()`` which also ignores them. - - Args: - columns: Column selection config. - all_info: If ``True``, equivalent to ``ColumnConfig(system_tags=True)`` - for this method. - - Returns: - Tuple of ``(tag_schema, data_schema)``. - """ - columns_config = ColumnConfig.handle_config(columns, all_info=all_info) - tag_schema = self._tag_schema - if columns_config.system_tags: - source_id_col, record_id_col = system_tag_column_names(self._schema_hash()) - tag_schema = Schema( - {**dict(tag_schema), source_id_col: str, record_id_col: str} - ) - return tag_schema, self._data_schema -``` - -- [ ] **Step 7: Run the new tests** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py::TestSourceNodeColumnConfig -v -``` - -Expected: All 5 tests PASS. - -- [ ] **Step 8: Run the full source_node test file to check for regressions** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py -v -``` - -Expected: All tests PASS. - -- [ ] **Step 9: Commit** - -```bash -git add src/orcapod/core/nodes/source_node.py tests/test_core/nodes/test_source_node.py -git commit -m "fix(source_node): SourceNodeBase.keys/output_schema honour system_tags column config" -``` - ---- - -## Task 5: Add `SourceJobNode.keys()` and `output_schema()` overrides (bound forwarding) - -**Files:** -- Modify: `tests/test_core/nodes/test_source_node.py` -- Modify: `src/orcapod/core/nodes/source_node.py` - -- [ ] **Step 1: Write the failing tests** - -Open `tests/test_core/nodes/test_source_node.py`. Append to the `TestSourceNodeColumnConfig` class: - -```python - def test_unbound_job_node_keys_default_unchanged(self, tag_schema, data_schema): - """Regression: unbound SourceJobNode.keys() still returns plain schema keys.""" - from orcapod.core.nodes.source_node import SourceJobNode - - job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - tag_keys, data_keys = job_node.keys() - assert tag_keys == ("id",) - assert data_keys == ("value",) - - def test_unbound_system_tag_names_match_bound(self, tag_schema, data_schema): - """Unbound SJN.keys(system_tags=True) returns the same names as the equivalent bound source.""" - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.core.sources.dict_source import DictSource - from orcapod.types import ColumnConfig - - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - unbound = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - bound = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src - ) - - unbound_keys = unbound.keys(columns=ColumnConfig(system_tags=True)) - bound_keys = bound.keys(columns=ColumnConfig(system_tags=True)) - assert unbound_keys == bound_keys - - def test_bound_keys_delegates_to_source(self, tag_schema, data_schema): - """Bound SourceJobNode.keys() delegates to bound_source.keys().""" - from unittest.mock import MagicMock - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.types import ColumnConfig - - mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) - mock_source.keys.return_value = (("id",), ("value",)) - - job_node = SourceJobNode(name="x", bound_source=mock_source) - cfg = ColumnConfig(system_tags=True) - job_node.keys(columns=cfg, all_info=False) - - mock_source.keys.assert_called_once_with(columns=cfg, all_info=False) - - def test_bound_keys_all_info_matches_source_directly(self, tag_schema, data_schema): - """Bound SJN.keys(all_info=True) == bound_source.keys(all_info=True).""" - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.core.sources.dict_source import DictSource - - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src - ) - - assert job_node.keys(all_info=True) == src.keys(all_info=True) - - def test_bound_output_schema_delegates_to_source(self, tag_schema, data_schema): - """Bound SourceJobNode.output_schema() delegates to bound_source.output_schema().""" - from unittest.mock import MagicMock - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.types import ColumnConfig - - mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) - - job_node = SourceJobNode(name="x", bound_source=mock_source) - cfg = ColumnConfig(system_tags=True) - job_node.output_schema(columns=cfg, all_info=False) - - mock_source.output_schema.assert_called_once_with(columns=cfg, all_info=False) - - def test_bound_output_schema_all_info_matches_source_directly(self, tag_schema, data_schema): - """Bound SJN.output_schema(all_info=True) == bound_source.output_schema(all_info=True).""" - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.core.sources.dict_source import DictSource - - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src - ) - - assert job_node.output_schema(all_info=True) == src.output_schema(all_info=True) -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py::TestSourceNodeColumnConfig -v -k "bound" -``` - -Expected: The `test_bound_keys_delegates_to_source` and `test_bound_output_schema_delegates_to_source` tests FAIL because `SourceJobNode` has no override — the mock's `keys` is never called. - -- [ ] **Step 3: Add `keys()` and `output_schema()` overrides to `SourceJobNode`** - -Open `src/orcapod/core/nodes/source_node.py`. Inside `SourceJobNode`, find the `as_table()` override (currently around line 524). Add the two new overrides **before** `as_table()`: - -```python - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Return ``(tag_keys, data_keys)``, delegating to ``bound_source`` when set. - - When ``bound_source`` is present, this is a transparent pass-through to - ``bound_source.keys(columns=columns, all_info=all_info)`` so callers get - the same result as querying the source directly. - - When unbound, delegates to ``SourceNodeBase.keys()`` which computes - system-tag column names from the declared schemas. - - Args: - columns: Column selection config. - all_info: If ``True``, include all available column groups. - - Returns: - Tuple of ``(tag_column_names, data_column_names)``. - """ - if self._bound_source is None: - return super().keys(columns=columns, all_info=all_info) - return self._bound_source.keys(columns=columns, all_info=all_info) - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return ``(tag_schema, data_schema)``, delegating to ``bound_source`` when set. - - When ``bound_source`` is present, this is a transparent pass-through to - ``bound_source.output_schema(columns=columns, all_info=all_info)`` so callers - get the same result as querying the source directly. - - When unbound, delegates to ``SourceNodeBase.output_schema()`` which includes - system-tag schema entries derived from the declared schemas. - - Args: - columns: Column selection config. - all_info: If ``True``, include all available column groups. - - Returns: - Tuple of ``(tag_schema, data_schema)``. - """ - if self._bound_source is None: - return super().output_schema(columns=columns, all_info=all_info) - return self._bound_source.output_schema(columns=columns, all_info=all_info) -``` - -- [ ] **Step 4: Run all new tests** - -```bash -uv run pytest tests/test_core/nodes/test_source_node.py::TestSourceNodeColumnConfig -v -``` - -Expected: All 11 tests PASS. - -- [ ] **Step 5: Run the entire test suite for the affected modules** - -```bash -uv run pytest tests/test_core/nodes/ tests/test_core/sources/ tests/test_utils/ -v -``` - -Expected: All tests PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/source_node.py tests/test_core/nodes/test_source_node.py -git commit -m "feat(source_node): SourceJobNode.keys/output_schema forward column_config to bound_source" -``` - ---- - -## Task 6: Full regression run and PR - -- [ ] **Step 1: Run the complete test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: All tests PASS, zero failures. - -- [ ] **Step 2: Push branch and open PR** - -```bash -git push -u origin eywalker/eng-516-sourcejobnode-and-sourcenode-should-accept-forward -``` - -Open a PR against the `dev` branch with title: -`fix(source_node): forward column_config/all_info in keys() and output_schema()` - -PR description must include: -- `Fixes ENG-516` -- Summary of changes: shared utilities added, SourceNodeBase fixed, SourceJobNode overrides added -- Reference to ENG-576 and ENG-577 as related follow-ups diff --git a/superpowers/plans/2026-06-08-eng-376-result-database-rename.md b/superpowers/plans/2026-06-08-eng-376-result-database-rename.md deleted file mode 100644 index 47c39fe96..000000000 --- a/superpowers/plans/2026-06-08-eng-376-result-database-rename.md +++ /dev/null @@ -1,154 +0,0 @@ -# ENG-376: result_database rename Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Rename `CachedFunctionPod._result_database` → `result_database` (and update all two definition/access sites) to fix the single FunctionNode/FunctionPod interface violation found in the ENG-376 audit. - -**Architecture:** Pure rename across three locations with no logic changes. (1) The `@property` definition on `CachedFunctionPod`. (2) The instance variable `self._result_database` on `_ResultDatabaseReader`. (3) The call site `self._cached_function_pod._result_database` inside `FunctionJobNode._fetch_joined_records()`. - -**Tech Stack:** Python 3.12, pytest via `uv run pytest` - ---- - -### Task 1: Write a failing test that asserts `result_database` is a public property - -The test accesses `cached_pod.result_database`. Before the rename this raises -`AttributeError` because the property is still named `_result_database`. - -**Files:** -- Modify: `tests/test_core/function_pod/test_cached_function_pod.py` - -- [ ] **Step 1: Add the test inside the existing `TestConstruction` class** - -Open `tests/test_core/function_pod/test_cached_function_pod.py`. - -The file already has a `TestConstruction` class and a `cached_pod` fixture that -creates a `CachedFunctionPod(function_pod=double_pod, result_database=cache_db)`. -Add the following method to `TestConstruction`: - -```python -def test_result_database_is_public(self, cached_pod): - """result_database must be accessible as a public property (no underscore prefix).""" - # Will raise AttributeError until the rename in cached_function_pod.py is made. - db = cached_pod.result_database - assert db is not None -``` - -- [ ] **Step 2: Run the test to confirm it fails** - -```bash -uv run pytest tests/test_core/function_pod/test_cached_function_pod.py::TestConstruction::test_result_database_is_public -v -``` - -Expected output (before the rename): -``` -FAILED ... AttributeError: 'CachedFunctionPod' object has no attribute 'result_database' -``` - ---- - -### Task 2: Make the rename in all three locations - -All three changes must be made together — they are a single logical unit. Making -only one or two will leave the code broken. - -**Files:** -- Modify: `src/orcapod/core/cached_function_pod.py` (rename `@property`) -- Modify: `src/orcapod/core/nodes/function_node.py` (rename in `_ResultDatabaseReader.__init__`; update call site in `FunctionJobNode._fetch_joined_records()`) - -- [ ] **Step 3: Rename the property on `CachedFunctionPod`** - -In `src/orcapod/core/cached_function_pod.py`, change: - -```python - @property - def _result_database(self) -> ArrowDatabaseProtocol: - """The underlying result database (for FunctionNode access).""" - return self._cache.result_database -``` - -to: - -```python - @property - def result_database(self) -> ArrowDatabaseProtocol: - """The underlying result database.""" - return self._cache.result_database -``` - -- [ ] **Step 4: Rename the instance variable on `_ResultDatabaseReader`** - -In `src/orcapod/core/nodes/function_node.py`, inside `_ResultDatabaseReader.__init__`, -change: - -```python - def __init__( - self, - result_database: ArrowDatabaseProtocol, - record_path: tuple[str, ...], - ) -> None: - self._result_database = result_database - self._record_path = record_path -``` - -to: - -```python - def __init__( - self, - result_database: ArrowDatabaseProtocol, - record_path: tuple[str, ...], - ) -> None: - self.result_database = result_database - self._record_path = record_path -``` - -- [ ] **Step 5: Update the call site in `FunctionJobNode._fetch_joined_records()`** - -In the same file (`function_node.py`), inside `_fetch_joined_records()`, change: - -```python - results = self._cached_function_pod._result_database.get_all_records( - self._cached_function_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) -``` - -to: - -```python - results = self._cached_function_pod.result_database.get_all_records( - self._cached_function_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) -``` - -- [ ] **Step 6: Run the new test to verify it now passes** - -```bash -uv run pytest tests/test_core/function_pod/test_cached_function_pod.py::TestConstruction::test_result_database_is_public -v -``` - -Expected output: -``` -PASSED -``` - -- [ ] **Step 7: Run the full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -v -``` - -Expected output: all tests pass. If any test fails with `AttributeError: '_result_database'`, -that test was directly accessing the private attribute and must be updated to use -`result_database` instead. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/core/cached_function_pod.py \ - src/orcapod/core/nodes/function_node.py \ - tests/test_core/function_pod/test_cached_function_pod.py -git commit -m "refactor(eng-376): rename CachedFunctionPod._result_database to result_database" -``` diff --git a/superpowers/plans/2026-06-08-eng-389-empty-container-type-inference.md b/superpowers/plans/2026-06-08-eng-389-empty-container-type-inference.md deleted file mode 100644 index 9bb89f99e..000000000 --- a/superpowers/plans/2026-06-08-eng-389-empty-container-type-inference.md +++ /dev/null @@ -1,338 +0,0 @@ -# ENG-389: Empty Container Type Inference Fix — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `list[Any]` and `dict[Any, Any]` (inferred from empty containers) convertible to Arrow types by mapping `typing.Any` ↔ `pa.null()` in `UniversalTypeConverter`. - -**Architecture:** Two targeted edits to `universal_converter.py`: add `Any: pa.null()` to the type map (forward path) and add an explicit `pa.null()` check in `_convert_arrow_to_python` (reverse path). Remove one dead-code block. No inference changes. - -**Tech Stack:** Python, PyArrow, `uv run pytest`. - -**Spec:** `superpowers/specs/2026-06-08-eng-389-empty-container-type-inference.md` - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/semantic_types/universal_converter.py` | Modify | Add `Any: pa.null()` to type map; add `pa.null()` case in `_convert_arrow_to_python`; remove dead `Any` hint block | -| `tests/test_semantic_types/test_universal_converter.py` | Modify | Add 6 converter/round-trip tests | -| `tests/test_semantic_types/test_pydata_utils.py` | Modify | Add 2 inference documentation tests | -| `DESIGN_ISSUES.md` | Modify | Add resolved entry for ENG-389 | - ---- - -## Task 1: Write failing converter tests - -**Files:** -- Modify: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Append the six new tests to the end of the file** - -Open `tests/test_semantic_types/test_universal_converter.py` and append: - -```python -# --------------------------------------------------------------------------- -# ENG-389: Any <-> pa.null() round-trip -# --------------------------------------------------------------------------- - -from typing import Any - - -def test_any_to_arrow_type(): - """typing.Any maps to pa.null().""" - assert universal_converter.python_type_to_arrow_type(Any) == pa.null() - - -def test_list_any_to_arrow_type(): - """list[Any] maps to pa.large_list(pa.null()).""" - assert ( - universal_converter.python_type_to_arrow_type(list[Any]) - == pa.large_list(pa.null()) - ) - - -def test_dict_any_any_to_arrow_type(): - """dict[Any, Any] maps to pa.large_list(pa.struct([("key", pa.null()), ("value", pa.null())])).""" - expected = pa.large_list( - pa.struct([("key", pa.null()), ("value", pa.null())]) - ) - assert universal_converter.python_type_to_arrow_type(dict[Any, Any]) == expected - - -def test_null_arrow_to_any_python_type(): - """pa.null() maps back to typing.Any.""" - assert universal_converter.arrow_type_to_python_type(pa.null()) is Any - - -def test_empty_container_inference_to_arrow_no_error(): - """Inferring schema from empty containers and converting to Arrow does not raise.""" - from orcapod.semantic_types.pydata_utils import infer_python_schema_from_pylist_data - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - schema = infer_python_schema_from_pylist_data([{"items": [], "meta": {}}]) - converter = UniversalTypeConverter() - # Must not raise ValueError: Unsupported Python type: typing.Any - arrow_schema = converter.python_schema_to_arrow_schema(schema) - assert "items" in [f.name for f in arrow_schema] - assert "meta" in [f.name for f in arrow_schema] - - -def test_pyarrow_empty_list_with_null_type(): - """PyArrow accepts empty-list values for pa.large_list(pa.null()) columns.""" - schema = pa.schema([ - pa.field("items", pa.large_list(pa.null())), - pa.field("meta", pa.large_list(pa.struct([("key", pa.null()), ("value", pa.null())]))), - ]) - table = pa.Table.from_pylist([{"items": [], "meta": []}], schema=schema) - assert table.num_rows == 1 - assert table.schema.field("items").type == pa.large_list(pa.null()) -``` - -- [ ] **Step 2: Run the new tests to confirm they fail** - -``` -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_any_to_arrow_type tests/test_semantic_types/test_universal_converter.py::test_list_any_to_arrow_type tests/test_semantic_types/test_universal_converter.py::test_dict_any_any_to_arrow_type tests/test_semantic_types/test_universal_converter.py::test_null_arrow_to_any_python_type tests/test_semantic_types/test_universal_converter.py::test_empty_container_inference_to_arrow_no_error tests/test_semantic_types/test_universal_converter.py::test_pyarrow_empty_list_with_null_type -v -``` - -Expected: all 6 FAIL. The first four fail with `AssertionError` or `ValueError: Unsupported Python type: typing.Any`. The fifth fails with `ValueError`. The sixth may pass (PyArrow assumption check) — if it does, that is acceptable. - ---- - -## Task 2: Implement the fix in `universal_converter.py` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` - -### Change A — add `Any: pa.null()` to the type map - -- [ ] **Step 1: Open `_get_python_to_arrow_map()` (around line 61) and add the `Any` entry** - -Find this block: - -```python - _PYTHON_TO_ARROW_MAP = { - # Python built-ins - int: pa.int64(), - float: pa.float64(), - str: pa.large_string(), # Use large_string by default for Polars compatibility - bool: pa.bool_(), - bytes: pa.large_binary(), # Use large_binary by default for Polars compatibility -``` - -Replace with: - -```python - _PYTHON_TO_ARROW_MAP = { - # Python built-ins - int: pa.int64(), - float: pa.float64(), - str: pa.large_string(), # Use large_string by default for Polars compatibility - bool: pa.bool_(), - bytes: pa.large_binary(), # Use large_binary by default for Polars compatibility - # typing.Any — used when element type is unknown (e.g. inferred from empty containers) - Any: pa.null(), -``` - -### Change B — add `pa.null()` case to `_convert_arrow_to_python()` - -- [ ] **Step 2: Open `_convert_arrow_to_python()` (around line 496) and add the null check** - -Find this block at the start of `_convert_arrow_to_python`: - -```python - # Handle basic types - if pa.types.is_integer(arrow_type): - return int - elif pa.types.is_floating(arrow_type): - return float -``` - -Replace with: - -```python - # Handle null type — maps to Any (unknown element type, e.g. from empty containers) - if pa.types.is_null(arrow_type): - return Any - - # Handle basic types - if pa.types.is_integer(arrow_type): - return int - elif pa.types.is_floating(arrow_type): - return float -``` - -### Change C — remove dead `Any` hint block from `_convert_python_to_arrow()` - -- [ ] **Step 3: Remove the now-unreachable `Any` hint in `_convert_python_to_arrow()` (around lines 423–432)** - -Find and remove this block entirely: - -```python - hint = "" - if python_type is Any: - hint = ( - " Hint: typing.Any usually appears when an Arrow type had " - "no mapping in arrow_type_to_python_type (check warnings). " - "It can also come from schema inference on empty containers " - "(e.g. {} infers as dict[Any, Any])." - ) - raise ValueError( - f"Unsupported Python type: {python_type}.{hint}" - ) -``` - -Replace with: - -```python - raise ValueError(f"Unsupported Python type: {python_type}.") -``` - -- [ ] **Step 4: Run all six new tests to confirm they now pass** - -``` -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_any_to_arrow_type tests/test_semantic_types/test_universal_converter.py::test_list_any_to_arrow_type tests/test_semantic_types/test_universal_converter.py::test_dict_any_any_to_arrow_type tests/test_semantic_types/test_universal_converter.py::test_null_arrow_to_any_python_type tests/test_semantic_types/test_universal_converter.py::test_empty_container_inference_to_arrow_no_error tests/test_semantic_types/test_universal_converter.py::test_pyarrow_empty_list_with_null_type -v -``` - -Expected: all 6 PASS. - -- [ ] **Step 5: Run the full `test_universal_converter.py` suite to check for regressions** - -``` -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -``` - -Expected: all existing tests still PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py \ - tests/test_semantic_types/test_universal_converter.py -git commit -m "fix(type-inference): map Any to pa.null() in UniversalTypeConverter - -Empty containers ([] / {}) infer element type as typing.Any, which previously -raised ValueError in python_type_to_arrow_type. Now Any maps to pa.null() (Arrow's -canonical unknown-type), and pa.null() maps back to Any on the reverse path. - -Closes ENG-389" -``` - ---- - -## Task 3: Add inference documentation tests to `test_pydata_utils.py` - -These tests document that inference of empty containers already yields `list[Any]` / -`dict[Any, Any]` (the inference is correct; the fix was in the converter). They should -pass immediately. - -**Files:** -- Modify: `tests/test_semantic_types/test_pydata_utils.py` - -- [ ] **Step 1: Append the two new tests to the end of the file** - -```python -# --------------------------------------------------------------------------- -# ENG-389: empty container inference produces list[Any] / dict[Any, Any] -# --------------------------------------------------------------------------- - -from typing import Any - - -def test_infer_empty_list_schema(): - """A field whose only value is [] infers as list[Any].""" - schema = pydata_utils.infer_python_schema_from_pylist_data([{"items": []}]) - assert schema["items"] == list[Any] - - -def test_infer_empty_dict_schema(): - """A field whose only value is {} infers as dict[Any, Any].""" - schema = pydata_utils.infer_python_schema_from_pylist_data([{"meta": {}}]) - assert schema["meta"] == dict[Any, Any] -``` - -- [ ] **Step 2: Run the two new tests to confirm they pass** - -``` -uv run pytest tests/test_semantic_types/test_pydata_utils.py::test_infer_empty_list_schema tests/test_semantic_types/test_pydata_utils.py::test_infer_empty_dict_schema -v -``` - -Expected: both PASS (inference already behaves correctly). - -- [ ] **Step 3: Run the full `test_pydata_utils.py` suite to check for regressions** - -``` -uv run pytest tests/test_semantic_types/test_pydata_utils.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_semantic_types/test_pydata_utils.py -git commit -m "test(type-inference): document empty container inference as list[Any]/dict[Any, Any] - -ENG-389" -``` - ---- - -## Task 4: Update `DESIGN_ISSUES.md` - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Add a new resolved entry** - -Append the following section to `DESIGN_ISSUES.md` (after the last existing entry): - -```markdown ---- - -## `src/orcapod/semantic_types/universal_converter.py` - -### UC1 — `python_type_to_arrow_type` raised on `typing.Any` from empty-container inference -**Status:** resolved -**Severity:** medium -**Issue:** ENG-389 - -`_infer_list_type` and `_infer_dict_type` in `pydata_utils.py` return `list[Any]` / -`dict[Any, Any]` when all sampled containers are empty (no elements to inspect). Passing -these inferred types to `python_type_to_arrow_type` raised `ValueError: Unsupported -Python type: typing.Any`. - -**Fix:** Added `Any: pa.null()` to `_PYTHON_TO_ARROW_MAP` (forward path) and an explicit -`pa.types.is_null → Any` check to `_convert_arrow_to_python` (reverse path). `pa.null()` -is Arrow's canonical "unknown/no-type" marker; empty containers have no elements to -validate, so the encoding is semantically correct. The now-unreachable `Any`-specific hint -in the error branch was removed. -``` - -- [ ] **Step 2: Run the full semantic-types test suite to confirm nothing regressed** - -``` -uv run pytest tests/test_semantic_types/ -v -``` - -Expected: all tests PASS. - -- [ ] **Step 3: Commit** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs(design-issues): log ENG-389 Any→pa.null() fix as resolved" -``` - ---- - -## Final verification - -- [ ] **Run the full test suite** - -``` -uv run pytest tests/ -x -q -``` - -Expected: all tests pass, no errors. diff --git a/superpowers/plans/2026-06-08-eng-571-tag-columns-normalization.md b/superpowers/plans/2026-06-08-eng-571-tag-columns-normalization.md deleted file mode 100644 index c0ea1b8d1..000000000 --- a/superpowers/plans/2026-06-08-eng-571-tag-columns-normalization.md +++ /dev/null @@ -1,931 +0,0 @@ -# `tag_columns` Bare-String Normalization Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a shared `_normalize_column_list()` helper and apply it in every user-facing source constructor so that `tag_columns="session_id"` is treated identically to `tag_columns=["session_id"]`. - -**Architecture:** A private helper in `schema_utils.py` normalizes bare strings to single-element lists and rejects non-string, non-iterable inputs with a `TypeError`. Normalization is applied only at user-facing source `__init__` methods — `SourceStreamBuilder.build()` is an internal contract and is not changed. - -**Tech Stack:** Python 3.11+, PyArrow, Polars (DataFrameSource only), pytest, uv - ---- - -## File Map - -| Action | File | Change | -|---|---|---| -| Modify | `src/orcapod/utils/schema_utils.py` | Add `_normalize_column_list()` | -| Modify | `src/orcapod/core/sources/data_frame_source.py` | Replace inline str-check with helper | -| Modify | `src/orcapod/core/sources/arrow_table_source.py` | Add normalization + update type hint | -| Modify | `src/orcapod/core/sources/csv_source.py` | Add normalization + update type hint | -| Modify | `src/orcapod/core/sources/dict_source.py` | Add normalization + update type hint | -| Modify | `src/orcapod/core/sources/delta_table_source.py` | Add normalization + update type hint | -| Modify | `src/orcapod/core/sources/db_table_source.py` | Replace `list(tag_columns)` + update type hint | -| Modify | `src/orcapod/core/sources/sqlite_table_source.py` | Replace `list(tag_columns)` + update type hint | -| Modify | `src/orcapod/core/sources/spiraldb_table_source.py` | Replace `list(tag_columns)` + update type hint | -| Modify | `src/orcapod/core/sources/postgresql_table_source.py` | Update type hint only | -| Create | `tests/test_core/sources/test_tag_columns_normalization.py` | All normalization tests | - ---- - -## Task 1: `_normalize_column_list` helper - -**Files:** -- Modify: `src/orcapod/utils/schema_utils.py` -- Test: `tests/test_core/sources/test_tag_columns_normalization.py` - -- [ ] **Step 1: Create the test file with failing helper unit tests** - -Create `tests/test_core/sources/test_tag_columns_normalization.py` with this content: - -```python -"""Tests for tag_columns bare-string normalization (ENG-571). - -Covers: -- _normalize_column_list: bare string, list, tuple, empty, invalid inputs -- Per-source integration: all user-facing sources accept bare string identically - to a single-element list. -""" -from __future__ import annotations - -import re -import sqlite3 -from collections.abc import Iterator -from typing import Any - -import pyarrow as pa -import pytest - -from orcapod.utils.schema_utils import _normalize_column_list - - -# --------------------------------------------------------------------------- -# Helper unit tests -# --------------------------------------------------------------------------- - - -class TestNormalizeColumnList: - def test_bare_string_returns_single_element_list(self): - assert _normalize_column_list("session_id") == ["session_id"] - - def test_single_element_list_unchanged(self): - assert _normalize_column_list(["session_id"]) == ["session_id"] - - def test_multi_element_list_unchanged(self): - assert _normalize_column_list(["a", "b", "c"]) == ["a", "b", "c"] - - def test_tuple_returns_list(self): - assert _normalize_column_list(("a", "b")) == ["a", "b"] - - def test_empty_list_returns_empty_list(self): - assert _normalize_column_list([]) == [] - - def test_integer_raises_type_error(self): - with pytest.raises(TypeError, match="tag_columns must be a string or iterable"): - _normalize_column_list(42) - - def test_float_raises_type_error(self): - with pytest.raises(TypeError, match="tag_columns must be a string or iterable"): - _normalize_column_list(3.14) - - def test_list_with_non_string_element_raises_type_error(self): - with pytest.raises(TypeError, match="All tag_columns elements must be strings"): - _normalize_column_list([1, "b"]) - - def test_list_with_all_non_string_elements_raises_type_error(self): - with pytest.raises(TypeError, match="All tag_columns elements must be strings"): - _normalize_column_list([1, 2, 3]) -``` - -- [ ] **Step 2: Run helper tests — expect ImportError / AttributeError (function not yet defined)** - -```bash -cd orcapod-python && uv run pytest tests/test_core/sources/test_tag_columns_normalization.py::TestNormalizeColumnList -v -``` - -Expected: all 9 tests fail with `ImportError: cannot import name '_normalize_column_list'`. - -- [ ] **Step 3: Add `_normalize_column_list` to `schema_utils.py`** - -At the bottom of `src/orcapod/utils/schema_utils.py`, append: - -```python -def _normalize_column_list(value: Any) -> list[str]: - """Normalize a column-list argument to a plain list of strings. - - Accepts a bare string (wraps it in a list), any iterable of strings - (converts to list), or raises ``TypeError`` for non-string non-iterable - inputs or iterables that contain non-string elements. - - Args: - value: A single column name (``str``) or an iterable of column names. - - Returns: - A list of column name strings. - - Raises: - TypeError: If ``value`` is not a ``str`` or iterable, or if any - element of the iterable is not a ``str``. - """ - if isinstance(value, str): - return [value] - try: - result = list(value) - except TypeError: - raise TypeError( - f"tag_columns must be a string or iterable of strings, " - f"got {type(value).__name__!r}" - ) - bad = [x for x in result if not isinstance(x, str)] - if bad: - raise TypeError( - f"All tag_columns elements must be strings; " - f"got {[type(x).__name__ for x in bad]!r}" - ) - return result -``` - -(`Any` is already imported at the top of `schema_utils.py` — no new import needed.) - -- [ ] **Step 4: Run helper tests — expect all to pass** - -```bash -uv run pytest tests/test_core/sources/test_tag_columns_normalization.py::TestNormalizeColumnList -v -``` - -Expected: 9 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/utils/schema_utils.py tests/test_core/sources/test_tag_columns_normalization.py -git commit -m "feat(sources): add _normalize_column_list helper for tag_columns normalization" -``` - ---- - -## Task 2: Integration tests for all source types - -**Files:** -- Modify: `tests/test_core/sources/test_tag_columns_normalization.py` - -Add the integration tests below to the existing test file. These test that each user-facing source accepts `tag_columns="col_name"` identically to `tag_columns=["col_name"]`. Most will **fail** until the source fixes land in Tasks 3–5 — that is expected. - -- [ ] **Step 1: Append integration tests to `test_tag_columns_normalization.py`** - -Add after the `TestNormalizeColumnList` class: - -```python -# --------------------------------------------------------------------------- -# Shared Arrow table fixture -# --------------------------------------------------------------------------- - - -def _make_table() -> pa.Table: - """Two-column table: session_id (tag candidate) + value (data).""" - return pa.table( - { - "session_id": pa.array(["s1", "s2"], type=pa.large_string()), - "value": pa.array([10, 20], type=pa.int64()), - } - ) - - -# --------------------------------------------------------------------------- -# ArrowTableSource -# --------------------------------------------------------------------------- - - -class TestArrowTableSourceTagColumns: - def test_bare_string_same_as_list(self): - from orcapod.core.sources import ArrowTableSource - - t = _make_table() - src_str = ArrowTableSource(table=t, tag_columns="session_id", infer_nullable=True) - src_list = ArrowTableSource(table=t, tag_columns=["session_id"], infer_nullable=True) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_tuple_accepted(self): - from orcapod.core.sources import ArrowTableSource - - t = _make_table() - src = ArrowTableSource(table=t, tag_columns=("session_id",), infer_nullable=True) - tag_keys, _ = src.keys() - assert "session_id" in tag_keys - - def test_invalid_type_raises(self): - from orcapod.core.sources import ArrowTableSource - - with pytest.raises(TypeError): - ArrowTableSource(table=_make_table(), tag_columns=42, infer_nullable=True) - - -# --------------------------------------------------------------------------- -# DataFrameSource -# --------------------------------------------------------------------------- - - -class TestDataFrameSourceTagColumns: - def test_bare_string_same_as_list(self): - from orcapod.core.sources import DataFrameSource - - data = {"session_id": ["s1", "s2"], "value": [10, 20]} - src_str = DataFrameSource(data=data, tag_columns="session_id") - src_list = DataFrameSource(data=data, tag_columns=["session_id"]) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_tuple_accepted(self): - from orcapod.core.sources import DataFrameSource - - src = DataFrameSource( - data={"session_id": ["s1"], "value": [10]}, - tag_columns=("session_id",), - ) - tag_keys, _ = src.keys() - assert "session_id" in tag_keys - - def test_invalid_type_raises(self): - from orcapod.core.sources import DataFrameSource - - with pytest.raises(TypeError): - DataFrameSource(data={"session_id": ["s1"], "value": [1]}, tag_columns=42) - - -# --------------------------------------------------------------------------- -# CSVSource -# --------------------------------------------------------------------------- - - -class TestCSVSourceTagColumns: - def test_bare_string_same_as_list(self, tmp_path): - from orcapod.core.sources import CSVSource - - p = tmp_path / "data.csv" - p.write_text("session_id,value\ns1,10\ns2,20\n") - src_str = CSVSource(file_path=str(p), tag_columns="session_id") - src_list = CSVSource(file_path=str(p), tag_columns=["session_id"]) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_tuple_accepted(self, tmp_path): - from orcapod.core.sources import CSVSource - - p = tmp_path / "data.csv" - p.write_text("session_id,value\ns1,10\n") - src = CSVSource(file_path=str(p), tag_columns=("session_id",)) - tag_keys, _ = src.keys() - assert "session_id" in tag_keys - - def test_invalid_type_raises(self, tmp_path): - from orcapod.core.sources import CSVSource - - p = tmp_path / "data.csv" - p.write_text("session_id,value\ns1,10\n") - with pytest.raises(TypeError): - CSVSource(file_path=str(p), tag_columns=42) - - -# --------------------------------------------------------------------------- -# DictSource -# --------------------------------------------------------------------------- - - -class TestDictSourceTagColumns: - _DATA = [{"session_id": "s1", "value": 10}, {"session_id": "s2", "value": 20}] - - def test_bare_string_same_as_list(self): - from orcapod.core.sources import DictSource - - src_str = DictSource(data=self._DATA, tag_columns="session_id") - src_list = DictSource(data=self._DATA, tag_columns=["session_id"]) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_tuple_accepted(self): - from orcapod.core.sources import DictSource - - src = DictSource(data=self._DATA, tag_columns=("session_id",)) - tag_keys, _ = src.keys() - assert "session_id" in tag_keys - - def test_invalid_type_raises(self): - from orcapod.core.sources import DictSource - - with pytest.raises(TypeError): - DictSource(data=self._DATA, tag_columns=42) - - -# --------------------------------------------------------------------------- -# DeltaTableSource (skipped when deltalake not installed) -# --------------------------------------------------------------------------- - - -class TestDeltaTableSourceTagColumns: - @pytest.fixture - def delta_path(self, tmp_path): - deltalake = pytest.importorskip("deltalake") - t = pa.table( - { - "session_id": pa.array(["s1", "s2"], type=pa.large_string()), - "value": pa.array([10, 20], type=pa.int64()), - } - ) - dest = tmp_path / "delta" - deltalake.write_deltalake(str(dest), t) - return dest - - def test_bare_string_same_as_list(self, delta_path): - from orcapod.core.sources import DeltaTableSource - - src_str = DeltaTableSource(delta_table_path=delta_path, tag_columns="session_id") - src_list = DeltaTableSource(delta_table_path=delta_path, tag_columns=["session_id"]) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_invalid_type_raises(self, delta_path): - from orcapod.core.sources import DeltaTableSource - - with pytest.raises(TypeError): - DeltaTableSource(delta_table_path=delta_path, tag_columns=42) - - -# --------------------------------------------------------------------------- -# DBTableSource (via MockDBConnector — no external DB required) -# --------------------------------------------------------------------------- - - -class _MockConnector: - """Minimal in-memory DBConnectorProtocol for normalization tests.""" - - _TABLE = pa.table( - { - "session_id": pa.array(["s1", "s2"], type=pa.large_string()), - "value": pa.array([10, 20], type=pa.int64()), - } - ) - - def get_table_names(self) -> list[str]: - return ["events"] - - def get_pk_columns(self, table_name: str) -> list[str]: - return ["session_id"] - - def get_column_info(self, table_name: str): - from orcapod.types import ColumnInfo - return [ColumnInfo(name=f.name, arrow_type=f.type) for f in self._TABLE.schema] - - def iter_batches(self, query: str, params: Any = None, batch_size: int = 1000) -> Iterator[pa.RecordBatch]: - m = re.search(r'FROM\s+"?(\w+)"?', query, re.IGNORECASE) - if m and m.group(1) == "events": - yield from self._TABLE.to_batches() - - def create_table_if_not_exists(self, *a: Any, **kw: Any) -> None: - pass - - def upsert_records(self, *a: Any, **kw: Any) -> None: - pass - - def close(self) -> None: - pass - - -class TestDBTableSourceTagColumns: - def test_bare_string_same_as_list(self): - from orcapod.core.sources import DBTableSource - - src_str = DBTableSource( - connector=_MockConnector(), table_name="events", tag_columns="session_id" - ) - src_list = DBTableSource( - connector=_MockConnector(), table_name="events", tag_columns=["session_id"] - ) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_tuple_accepted(self): - from orcapod.core.sources import DBTableSource - - src = DBTableSource( - connector=_MockConnector(), table_name="events", tag_columns=("session_id",) - ) - tag_keys, _ = src.keys() - assert "session_id" in tag_keys - - def test_invalid_type_raises(self): - from orcapod.core.sources import DBTableSource - - with pytest.raises(TypeError): - DBTableSource(connector=_MockConnector(), table_name="events", tag_columns=42) - - -# --------------------------------------------------------------------------- -# SQLiteTableSource -# --------------------------------------------------------------------------- - - -@pytest.fixture -def sqlite_db_path(tmp_path): - """SQLite DB with an 'events' table; session_id is the primary key.""" - db_path = tmp_path / "test.db" - conn = sqlite3.connect(str(db_path)) - conn.execute( - "CREATE TABLE events (session_id TEXT PRIMARY KEY, value INTEGER NOT NULL)" - ) - conn.executemany("INSERT INTO events VALUES (?, ?)", [("s1", 10), ("s2", 20)]) - conn.commit() - conn.close() - return str(db_path) - - -class TestSQLiteTableSourceTagColumns: - def test_bare_string_same_as_list(self, sqlite_db_path): - from orcapod.core.sources import SQLiteTableSource - - src_str = SQLiteTableSource( - db_path=sqlite_db_path, table_name="events", tag_columns="session_id" - ) - src_list = SQLiteTableSource( - db_path=sqlite_db_path, table_name="events", tag_columns=["session_id"] - ) - tag_keys_str, _ = src_str.keys() - tag_keys_list, _ = src_list.keys() - assert set(tag_keys_str) == set(tag_keys_list) == {"session_id"} - - def test_tuple_accepted(self, sqlite_db_path): - from orcapod.core.sources import SQLiteTableSource - - src = SQLiteTableSource( - db_path=sqlite_db_path, table_name="events", tag_columns=("session_id",) - ) - tag_keys, _ = src.keys() - assert "session_id" in tag_keys - - def test_invalid_type_raises(self, sqlite_db_path): - from orcapod.core.sources import SQLiteTableSource - - with pytest.raises(TypeError): - SQLiteTableSource( - db_path=sqlite_db_path, table_name="events", tag_columns=42 - ) -``` - -- [ ] **Step 2: Run integration tests — expect DataFrameSource to pass, all others to fail** - -```bash -uv run pytest tests/test_core/sources/test_tag_columns_normalization.py -v -``` - -Expected: -- `TestNormalizeColumnList` — all 9 PASS (from Task 1) -- `TestDataFrameSourceTagColumns` — all 3 PASS (already has inline fix) -- All other `Test*TagColumns` classes — FAIL (bare string iterates characters) - -- [ ] **Step 3: Commit the test file** - -```bash -git add tests/test_core/sources/test_tag_columns_normalization.py -git commit -m "test(sources): add tag_columns normalization integration tests (red)" -``` - ---- - -## Task 3: Fix `DataFrameSource` — replace inline check with helper - -**Files:** -- Modify: `src/orcapod/core/sources/data_frame_source.py` - -The current inline check (lines 54–56) works but duplicates logic we now have in the helper. This task refactors it. - -- [ ] **Step 1: Add import and replace inline check in `data_frame_source.py`** - -At the top of `src/orcapod/core/sources/data_frame_source.py`, the existing imports include: - -```python -from orcapod.utils import arrow_utils, polars_data_utils -``` - -Add the helper import on a new line after that: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Then replace lines 54–56 (the inline str check): - -```python - if isinstance(tag_columns, str): - tag_columns = [tag_columns] - tag_columns = list(tag_columns) -``` - -with: - -```python - tag_columns = _normalize_column_list(tag_columns) -``` - -The `tag_columns` parameter type annotation in the `__init__` signature is already `str | Collection[str]` — no change needed there. - -- [ ] **Step 2: Run DataFrameSource tests — expect all to pass** - -```bash -uv run pytest tests/test_core/sources/test_tag_columns_normalization.py::TestDataFrameSourceTagColumns -v -``` - -Expected: 3 PASS (behavior unchanged — this is a pure refactor). - -- [ ] **Step 3: Run broader DataFrameSource tests to check for regressions** - -```bash -uv run pytest tests/test_core/sources/test_sources_comprehensive.py -v -k "DataFrameSource" -``` - -Expected: all existing DataFrameSource tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/core/sources/data_frame_source.py -git commit -m "refactor(sources): use _normalize_column_list in DataFrameSource" -``` - ---- - -## Task 4: Fix direct-pass sources — `ArrowTableSource`, `CSVSource`, `DictSource`, `DeltaTableSource` - -These sources pass `tag_columns` directly to `builder.build()` without any `list()` call. The fix is to normalize at the top of `__init__`. - -**Files:** -- Modify: `src/orcapod/core/sources/arrow_table_source.py` -- Modify: `src/orcapod/core/sources/csv_source.py` -- Modify: `src/orcapod/core/sources/dict_source.py` -- Modify: `src/orcapod/core/sources/delta_table_source.py` - -### `ArrowTableSource` - -- [ ] **Step 1: Update `arrow_table_source.py`** - -Add import after the existing `from orcapod.utils import arrow_utils` line: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` parameter type annotation in `__init__` from `Collection[str]` to `str | Collection[str]`: - -```python - def __init__( - self, - table: "pa.Table", - tag_columns: str | Collection[str] = (), - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - infer_nullable: bool = False, - **kwargs: Any, - ) -> None: -``` - -Add normalization immediately after `super().__init__(**kwargs)`, before the `if infer_nullable:` block: - -```python - tag_columns = _normalize_column_list(tag_columns) -``` - -### `CSVSource` - -- [ ] **Step 2: Update `csv_source.py`** - -Add import after the existing `from orcapod.utils import arrow_utils` line: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` type annotation: - -```python - def __init__( - self, - file_path: str, - tag_columns: str | Collection[str] = (), - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - source_id: str | None = None, - schema: pa.Schema | None = None, - **kwargs: Any, - ) -> None: -``` - -Add normalization as the first line after `super().__init__(...)`: - -```python - tag_columns = _normalize_column_list(tag_columns) -``` - -(Place it right after the `super().__init__(source_id=source_id, **kwargs)` call, before the `self._file_path = file_path` line.) - -### `DictSource` - -- [ ] **Step 3: Update `dict_source.py`** - -Add import after the existing `from orcapod.utils import arrow_utils` line: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` type annotation: - -```python - def __init__( - self, - data: Collection[Mapping[str, DataValue]], - tag_columns: str | Collection[str] = (), - system_tag_columns: Collection[str] = (), - data_schema: SchemaLike | pa.Schema | None = None, - source_id: str | None = None, - **kwargs: Any, - ) -> None: -``` - -Add normalization immediately after `super().__init__(source_id=source_id, **kwargs)`, before the `if isinstance(data_schema, pa.Schema):` block: - -```python - tag_columns = _normalize_column_list(tag_columns) -``` - -### `DeltaTableSource` - -- [ ] **Step 4: Update `delta_table_source.py`** - -Add a new import for the helper. The file currently has no `orcapod.utils` imports. Add after the last `from orcapod...` import line: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` type annotation: - -```python - def __init__( - self, - delta_table_path: PathLike, - tag_columns: str | Collection[str] = (), - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - source_id: str | None = None, - **kwargs: Any, - ) -> None: -``` - -Add normalization as the first line after the `super().__init__(source_id=source_id, **kwargs)` call: - -```python - tag_columns = _normalize_column_list(tag_columns) -``` - -(Place it right before the `self._delta_table_path = resolved` assignment.) - -- [ ] **Step 5: Run integration tests for these four sources — expect all to pass now** - -```bash -uv run pytest tests/test_core/sources/test_tag_columns_normalization.py::TestArrowTableSourceTagColumns tests/test_core/sources/test_tag_columns_normalization.py::TestCSVSourceTagColumns tests/test_core/sources/test_tag_columns_normalization.py::TestDictSourceTagColumns tests/test_core/sources/test_tag_columns_normalization.py::TestDeltaTableSourceTagColumns -v -``` - -Expected: all tests PASS (DeltaTableSource tests skip if `deltalake` not installed — that's fine). - -- [ ] **Step 6: Run existing source tests to check for regressions** - -```bash -uv run pytest tests/test_core/sources/test_sources_comprehensive.py tests/test_core/sources/test_sources.py -v -``` - -Expected: all existing tests PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/sources/arrow_table_source.py \ - src/orcapod/core/sources/csv_source.py \ - src/orcapod/core/sources/dict_source.py \ - src/orcapod/core/sources/delta_table_source.py -git commit -m "feat(sources): normalize tag_columns in ArrowTableSource, CSVSource, DictSource, DeltaTableSource" -``` - ---- - -## Task 5: Fix pre-processed sources — `DBTableSource`, `PostgreSQLTableSource`, `SQLiteTableSource`, `SpiralDBTableSource` - -These sources call `list(tag_columns)` *before* delegating to the builder. Replace each with `_normalize_column_list(tag_columns)`. - -**Files:** -- Modify: `src/orcapod/core/sources/db_table_source.py` -- Modify: `src/orcapod/core/sources/postgresql_table_source.py` -- Modify: `src/orcapod/core/sources/sqlite_table_source.py` -- Modify: `src/orcapod/core/sources/spiraldb_table_source.py` - -### `DBTableSource` - -- [ ] **Step 1: Update `db_table_source.py`** - -Add import after `from orcapod.utils import arrow_utils`: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` type annotation in `__init__`: - -```python - def __init__( - self, - connector: DBConnectorProtocol, - table_name: str, - tag_columns: str | Collection[str] | None = None, - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - source_id: str | None = None, - label: str | None = None, - data_context: str | contexts.DataContext | None = None, - config: OrcapodConfig | None = None, - *, - _query: str | None = None, - ) -> None: -``` - -In Step 2 of `__init__` (resolving tag columns), replace: - -```python - else: - resolved_tag_columns = list(tag_columns) -``` - -with: - -```python - else: - resolved_tag_columns = _normalize_column_list(tag_columns) -``` - -### `PostgreSQLTableSource` - -- [ ] **Step 2: Update `postgresql_table_source.py` — type hint only** - -Update the `tag_columns` type annotation in `__init__`. No logic change: - -```python - def __init__( - self, - dsn: str, - table_name: str, - tag_columns: str | Collection[str] | None = None, - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - source_id: str | None = None, - label: str | None = None, - data_context: str | contexts.DataContext | None = None, - config: OrcapodConfig | None = None, - ) -> None: -``` - -Also update the `tag_columns` type in the docstring `Args` section from `Collection[str] | None` to `str | Collection[str] | None`. - -### `SQLiteTableSource` - -- [ ] **Step 3: Update `sqlite_table_source.py`** - -Add import. The file currently has no `orcapod.utils` imports. Add after the last `from orcapod...` import line: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` type annotation: - -```python - def __init__( - self, - db_path: str | os.PathLike, - table_name: str, - tag_columns: str | Collection[str] | None = None, - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - source_id: str | None = None, - label: str | None = None, - data_context: str | contexts.DataContext | None = None, - config: "OrcapodConfig | None" = None, - ) -> None: -``` - -In Step 3 (resolving tag columns), replace: - -```python - else: - resolved_tags = list(tag_columns) -``` - -with: - -```python - else: - resolved_tags = _normalize_column_list(tag_columns) -``` - -### `SpiralDBTableSource` - -- [ ] **Step 4: Update `spiraldb_table_source.py`** - -Add import. The file has no `orcapod.utils` imports. Add after the last `from orcapod...` import line: - -```python -from orcapod.utils.schema_utils import _normalize_column_list -``` - -Update the `tag_columns` type annotation: - -```python - def __init__( - self, - project_id: str, - table_name: str, - dataset: str = "default", - tag_columns: str | Collection[str] | None = None, - system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, - source_id: str | None = None, - label: str | None = None, - data_context: "str | contexts.DataContext | None" = None, - config: "OrcapodConfig | None" = None, - overrides: dict[str, str] | None = None, - ) -> None: -``` - -Replace the `resolved_tags` assignment: - -```python - resolved_tags: list[str] | None = ( - list(tag_columns) if tag_columns is not None else None - ) -``` - -with: - -```python - resolved_tags: list[str] | None = ( - _normalize_column_list(tag_columns) if tag_columns is not None else None - ) -``` - -- [ ] **Step 5: Run DB source integration tests — expect all to pass** - -```bash -uv run pytest tests/test_core/sources/test_tag_columns_normalization.py::TestDBTableSourceTagColumns tests/test_core/sources/test_tag_columns_normalization.py::TestSQLiteTableSourceTagColumns -v -``` - -Expected: all 6 tests PASS. - -- [ ] **Step 6: Run existing DB source tests to check for regressions** - -```bash -uv run pytest tests/test_core/sources/test_db_table_source.py tests/test_core/sources/test_sqlite_table_source.py -v -``` - -Expected: all existing tests PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/sources/db_table_source.py \ - src/orcapod/core/sources/postgresql_table_source.py \ - src/orcapod/core/sources/sqlite_table_source.py \ - src/orcapod/core/sources/spiraldb_table_source.py -git commit -m "feat(sources): normalize tag_columns in DBTableSource, PostgreSQLTableSource, SQLiteTableSource, SpiralDBTableSource" -``` - ---- - -## Task 6: Full test suite verification - -- [ ] **Step 1: Run the complete test suite (excluding PostgreSQL integration tests)** - -```bash -uv run pytest tests/ -v --ignore=tests/test_core/sources/test_postgresql_table_source_integration.py -m "not postgres" -``` - -Expected: all tests PASS. No regressions anywhere. - -- [ ] **Step 2: Run the full normalization test file to confirm all tests green** - -```bash -uv run pytest tests/test_core/sources/test_tag_columns_normalization.py -v -``` - -Expected: all tests PASS (DeltaTableSource tests may skip if `deltalake` not installed — that is acceptable). diff --git a/superpowers/plans/2026-06-08-plt-964-persistent-config.md b/superpowers/plans/2026-06-08-plt-964-persistent-config.md deleted file mode 100644 index 1c27bf2c2..000000000 --- a/superpowers/plans/2026-06-08-plt-964-persistent-config.md +++ /dev/null @@ -1,969 +0,0 @@ -# PLT-964: Persistent OrcaPod Configuration System — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Restructure `OrcapodConfig` into a nested sectioned config (`HashingConfig` + `DisplayConfig`), add a `from_dict()` classmethod, update all consumers, and add `load_config()` for TOML-backed persistent configuration. - -**Architecture:** `config.py` is rewritten with three frozen dataclasses (`HashingConfig`, `DisplayConfig`, `OrcapodConfig`) and two new functions (`OrcapodConfig.from_dict()`, `load_config()`). Consumer files are updated in a single mechanical sweep to use new field paths. The file-loading layer uses Python's built-in `tomllib` (Python ≥3.11) — no new dependencies. - -**Tech Stack:** Python 3.11+, `tomllib` (stdlib), `dataclasses`, `logging`, `uv run pytest`. - -**Spec:** `superpowers/specs/2026-06-08-plt-964-persistent-config-design.md` - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/config.py` | Full rewrite | `HashingConfig`, `DisplayConfig`, nested `OrcapodConfig`, `from_dict()`, `load_config()` | -| `src/orcapod/__init__.py` | Modify | Export `HashingConfig`, `DisplayConfig`, `load_config` | -| `src/orcapod/core/sources/stream_builder.py` | Modify | `.schema_hash_n_char` → `.hashing.schema_n_char`; `.path_hash_n_char` → `.hashing.path_n_char` | -| `src/orcapod/core/nodes/source_node.py` | Modify | `.schema_hash_n_char` → `.hashing.schema_n_char` | -| `src/orcapod/core/operators/join.py` | Modify | `.system_tag_hash_n_char` → `.hashing.system_tag_n_char` (×3) | -| `src/orcapod/core/operators/merge_join.py` | Modify | `.system_tag_hash_n_char` → `.hashing.system_tag_n_char` (×2) | -| `src/orcapod/core/nodes/function_node.py` | Modify | `OrcapodConfig(**config_dict)` → `OrcapodConfig.from_dict(config_dict)` (×2) | -| `src/orcapod/core/nodes/operator_node.py` | Modify | `OrcapodConfig(**config_dict)` → `OrcapodConfig.from_dict(config_dict)` (×2) | -| `src/orcapod/utils/schema_utils.py` | Modify | Update docstring reference from `schema_hash_n_char` to `hashing.schema_n_char` | -| `tests/test_orcapod_config.py` | Full rewrite | New structural tests, `from_dict()` tests, `load_config()` tests | -| `tests/test_core/operators/test_operators.py` | Modify | `.system_tag_hash_n_char` → `.hashing.system_tag_n_char` (×2) | -| `tests/test_core/operators/test_merge_join.py` | Modify | `.system_tag_hash_n_char` → `.hashing.system_tag_n_char` (×1) | -| `tests/test_utils/test_schema_utils.py` | Modify | `DEFAULT_CONFIG.schema_hash_n_char` → `DEFAULT_CONFIG.hashing.schema_n_char` | - ---- - -## Task 1: Rewrite `config.py` and `test_orcapod_config.py` - -**Files:** -- Rewrite: `src/orcapod/config.py` -- Rewrite: `tests/test_orcapod_config.py` - -After this task the new config tests pass; tests that reference old flat field names in -other files will fail — that is expected and fixed in Task 2. - -- [ ] **Step 1: Rewrite `tests/test_orcapod_config.py`** - -Replace the entire file with: - -```python -"""Tests for HashingConfig, DisplayConfig, OrcapodConfig, and load_config (PLT-964).""" -from __future__ import annotations - - -class TestHashingConfig: - def test_defaults(self): - from orcapod.config import HashingConfig - - cfg = HashingConfig() - assert cfg.system_tag_n_char == 12 - assert cfg.schema_n_char == 12 - assert cfg.path_n_char == 20 - - def test_is_frozen(self): - import pytest - - from orcapod.config import HashingConfig - - cfg = HashingConfig() - with pytest.raises((AttributeError, TypeError)): - cfg.system_tag_n_char = 99 # type: ignore[misc] - - def test_merge_non_default_wins(self): - from orcapod.config import HashingConfig - - base = HashingConfig() - other = HashingConfig(system_tag_n_char=8) - merged = base.merge(other) - assert merged.system_tag_n_char == 8 - assert merged.schema_n_char == 12 # unchanged - - def test_merge_default_does_not_override(self): - from orcapod.config import HashingConfig - - base = HashingConfig(system_tag_n_char=6) - other = HashingConfig() # all defaults - merged = base.merge(other) - assert merged.system_tag_n_char == 6 # base value preserved - - -class TestDisplayConfig: - def test_defaults(self): - from orcapod.config import DisplayConfig - - cfg = DisplayConfig() - assert cfg.max_rows is None - assert cfg.show_meta_columns is False - assert cfg.show_source_columns is False - assert cfg.show_system_tag_columns is False - assert cfg.show_context_columns is False - - def test_is_frozen(self): - import pytest - - from orcapod.config import DisplayConfig - - cfg = DisplayConfig() - with pytest.raises((AttributeError, TypeError)): - cfg.max_rows = 10 # type: ignore[misc] - - def test_merge_non_default_wins(self): - from orcapod.config import DisplayConfig - - base = DisplayConfig() - other = DisplayConfig(max_rows=100, show_source_columns=True) - merged = base.merge(other) - assert merged.max_rows == 100 - assert merged.show_source_columns is True - assert merged.show_meta_columns is False # unchanged - - def test_merge_default_does_not_override(self): - from orcapod.config import DisplayConfig - - base = DisplayConfig(max_rows=50) - other = DisplayConfig() # all defaults - merged = base.merge(other) - assert merged.max_rows == 50 # base value preserved - - -class TestOrcapodConfig: - def test_defaults(self): - from orcapod.config import DisplayConfig, HashingConfig, OrcapodConfig - - cfg = OrcapodConfig() - assert cfg.hashing == HashingConfig() - assert cfg.display == DisplayConfig() - - def test_default_config_is_orcapod_config_instance(self): - from orcapod.config import DEFAULT_CONFIG, OrcapodConfig - - assert isinstance(DEFAULT_CONFIG, OrcapodConfig) - - def test_with_updates_hashing_section(self): - from orcapod.config import HashingConfig, OrcapodConfig - - cfg = OrcapodConfig() - updated = cfg.with_updates(hashing=HashingConfig(system_tag_n_char=8)) - assert updated.hashing.system_tag_n_char == 8 - assert cfg.hashing.system_tag_n_char == 12 # original unchanged - - def test_with_updates_display_section(self): - from orcapod.config import DisplayConfig, OrcapodConfig - - cfg = OrcapodConfig() - updated = cfg.with_updates(display=DisplayConfig(max_rows=50)) - assert updated.display.max_rows == 50 - assert cfg.display.max_rows is None # original unchanged - - def test_merge_hashing_non_default_wins(self): - from orcapod.config import HashingConfig, OrcapodConfig - - base = OrcapodConfig() - other = OrcapodConfig(hashing=HashingConfig(system_tag_n_char=8)) - merged = base.merge(other) - assert merged.hashing.system_tag_n_char == 8 - - def test_merge_hashing_default_does_not_override(self): - from orcapod.config import HashingConfig, OrcapodConfig - - base = OrcapodConfig(hashing=HashingConfig(system_tag_n_char=6)) - other = OrcapodConfig() - merged = base.merge(other) - assert merged.hashing.system_tag_n_char == 6 - - def test_merge_display_non_default_wins(self): - from orcapod.config import DisplayConfig, OrcapodConfig - - base = OrcapodConfig() - other = OrcapodConfig(display=DisplayConfig(max_rows=100)) - merged = base.merge(other) - assert merged.display.max_rows == 100 - - def test_merge_type_error(self): - import pytest - - from orcapod.config import OrcapodConfig - - cfg = OrcapodConfig() - with pytest.raises(TypeError): - cfg.merge("not a config") # type: ignore[arg-type] - - -class TestOrcapodConfigFromDict: - def test_empty_dict_returns_default(self): - from orcapod.config import DEFAULT_CONFIG, OrcapodConfig - - assert OrcapodConfig.from_dict({}) == DEFAULT_CONFIG - - def test_partial_hashing_section(self): - from orcapod.config import OrcapodConfig - - cfg = OrcapodConfig.from_dict({"hashing": {"system_tag_n_char": 8}}) - assert cfg.hashing.system_tag_n_char == 8 - assert cfg.hashing.schema_n_char == 12 # default preserved - - def test_full_sections_round_trip(self): - from orcapod.config import OrcapodConfig - - data = { - "hashing": {"system_tag_n_char": 8, "schema_n_char": 6, "path_n_char": 16}, - "display": {"max_rows": 100, "show_source_columns": True}, - } - cfg = OrcapodConfig.from_dict(data) - assert cfg.hashing.system_tag_n_char == 8 - assert cfg.hashing.schema_n_char == 6 - assert cfg.hashing.path_n_char == 16 - assert cfg.display.max_rows == 100 - assert cfg.display.show_source_columns is True - - def test_unknown_section_warns(self, caplog): - import logging - - from orcapod.config import OrcapodConfig - - with caplog.at_level(logging.WARNING, logger="orcapod.config"): - OrcapodConfig.from_dict({"typo_section": {"foo": 1}}) - assert "typo_section" in caplog.text - - def test_unknown_field_within_section_warns(self, caplog): - import logging - - from orcapod.config import OrcapodConfig - - with caplog.at_level(logging.WARNING, logger="orcapod.config"): - OrcapodConfig.from_dict({"display": {"max_rwos": 50}}) - assert "max_rwos" in caplog.text - - def test_source_path_appears_in_warning(self, caplog): - import logging - from pathlib import Path - - from orcapod.config import OrcapodConfig - - with caplog.at_level(logging.WARNING, logger="orcapod.config"): - OrcapodConfig.from_dict( - {"bad_section": {}}, source_path=Path("/some/config.toml") - ) - assert "/some/config.toml" in caplog.text - - -class TestLoadConfig: - def test_no_files_returns_default(self, tmp_path): - from orcapod.config import DEFAULT_CONFIG, load_config - - result = load_config( - project_config_path=tmp_path / "nonexistent.toml", - user_config_path=tmp_path / "nonexistent.toml", - ) - assert result == DEFAULT_CONFIG - - def test_user_global_config_applied(self, tmp_path): - from orcapod.config import load_config - - user_cfg = tmp_path / "config.toml" - user_cfg.write_text("[hashing]\nsystem_tag_n_char = 8\n") - result = load_config( - user_config_path=user_cfg, - project_config_path=tmp_path / "nonexistent.toml", - ) - assert result.hashing.system_tag_n_char == 8 - - def test_project_local_config_applied(self, tmp_path): - from orcapod.config import load_config - - project_cfg = tmp_path / "orcapod_config.toml" - project_cfg.write_text("[display]\nmax_rows = 50\n") - result = load_config( - project_config_path=project_cfg, - user_config_path=tmp_path / "nonexistent.toml", - ) - assert result.display.max_rows == 50 - - def test_project_local_wins_over_user_global(self, tmp_path): - from orcapod.config import load_config - - user_cfg = tmp_path / "user.toml" - user_cfg.write_text("[hashing]\nsystem_tag_n_char = 6\n") - project_cfg = tmp_path / "project.toml" - project_cfg.write_text("[hashing]\nsystem_tag_n_char = 10\n") - result = load_config(user_config_path=user_cfg, project_config_path=project_cfg) - assert result.hashing.system_tag_n_char == 10 - - def test_malformed_toml_raises_value_error(self, tmp_path): - import pytest - - from orcapod.config import load_config - - bad_cfg = tmp_path / "bad.toml" - bad_cfg.write_text("this is not valid toml ][[[") - with pytest.raises(ValueError, match=str(bad_cfg)): - load_config( - project_config_path=bad_cfg, - user_config_path=tmp_path / "nonexistent.toml", - ) - - def test_explicit_paths_used(self, tmp_path): - from orcapod.config import load_config - - cfg_file = tmp_path / "custom.toml" - cfg_file.write_text("[display]\nshow_source_columns = true\n") - result = load_config( - project_config_path=cfg_file, - user_config_path=tmp_path / "nonexistent.toml", - ) - assert result.display.show_source_columns is True - - -class TestOrcapodConfigTopLevelExport: - def test_can_import_orcapod_config(self): - from orcapod import OrcapodConfig # noqa: F401 - - def test_can_import_hashing_config(self): - from orcapod import HashingConfig # noqa: F401 - - def test_can_import_display_config(self): - from orcapod import DisplayConfig # noqa: F401 - - def test_can_import_load_config(self): - from orcapod import load_config # noqa: F401 - - def test_orcapod_config_in_all(self): - import orcapod - - assert "OrcapodConfig" in orcapod.__all__ - - def test_hashing_config_in_all(self): - import orcapod - - assert "HashingConfig" in orcapod.__all__ - - def test_display_config_in_all(self): - import orcapod - - assert "DisplayConfig" in orcapod.__all__ - - def test_load_config_in_all(self): - import orcapod - - assert "load_config" in orcapod.__all__ -``` - -- [ ] **Step 2: Run new tests to confirm they fail** - -```bash -uv run pytest tests/test_orcapod_config.py -v -``` - -Expected: `ImportError` on `HashingConfig`, `DisplayConfig`, `load_config` — the new names don't exist yet. - -- [ ] **Step 3: Rewrite `src/orcapod/config.py`** - -Replace the entire file with: - -```python -# config.py -from __future__ import annotations - -import logging -from dataclasses import dataclass, field, replace -from pathlib import Path -from typing import Any, Self - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class HashingConfig: - """Hash truncation length settings. - - Controls the number of hex characters used when truncating hashes for - system-tag column names, schema hashes, and database path scoping. - """ - - system_tag_n_char: int = 12 - schema_n_char: int = 12 - path_n_char: int = 20 - - def merge(self, other: "HashingConfig") -> "HashingConfig": - """Merge with another ``HashingConfig``; other takes precedence for non-default values. - - Args: - other: Config to merge in. Must be a ``HashingConfig`` instance. - - Returns: - New ``HashingConfig`` with non-default values from ``other`` applied. - - Raises: - TypeError: If ``other`` is not a ``HashingConfig`` instance. - """ - if not isinstance(other, HashingConfig): - raise TypeError("Can only merge with another HashingConfig instance") - defaults = HashingConfig() - updates = { - f: getattr(other, f) - for f in self.__dataclass_fields__ - if getattr(other, f) != getattr(defaults, f) - } - return replace(self, **updates) - - -@dataclass(frozen=True) -class DisplayConfig: - """Display and preview preference settings. - - Controls default row limits and column visibility when rendering streams - and tables. These values are consumed by output methods when no explicit - override is supplied by the caller. - """ - - max_rows: int | None = None - show_meta_columns: bool = False - show_source_columns: bool = False - show_system_tag_columns: bool = False - show_context_columns: bool = False - - def merge(self, other: "DisplayConfig") -> "DisplayConfig": - """Merge with another ``DisplayConfig``; other takes precedence for non-default values. - - Args: - other: Config to merge in. Must be a ``DisplayConfig`` instance. - - Returns: - New ``DisplayConfig`` with non-default values from ``other`` applied. - - Raises: - TypeError: If ``other`` is not a ``DisplayConfig`` instance. - """ - if not isinstance(other, DisplayConfig): - raise TypeError("Can only merge with another DisplayConfig instance") - defaults = DisplayConfig() - updates = { - f: getattr(other, f) - for f in self.__dataclass_fields__ - if getattr(other, f) != getattr(defaults, f) - } - return replace(self, **updates) - - -@dataclass(frozen=True) -class OrcapodConfig: - """Top-level immutable OrcaPod configuration. - - Groups all configuration into typed sections. Construct directly for - programmatic configuration, or use ``load_config()`` to load from TOML files. - """ - - hashing: HashingConfig = field(default_factory=HashingConfig) - display: DisplayConfig = field(default_factory=DisplayConfig) - - def with_updates(self, **kwargs: Any) -> Self: - """Create a new ``OrcapodConfig`` with updated section values. - - Args: - **kwargs: Section keyword arguments (e.g. ``hashing=HashingConfig(...)``). - - Returns: - New ``OrcapodConfig`` with the given sections replaced. - """ - return replace(self, **kwargs) - - def merge(self, other: "OrcapodConfig") -> "OrcapodConfig": - """Merge with another ``OrcapodConfig``; other takes precedence for non-default values. - - Merging is performed section by section via each section's own ``merge()``. - Within each section, fields that are non-default in ``other`` override the - corresponding fields in ``self``; fields at their default in ``other`` are - left unchanged. - - Args: - other: Config to merge in. Must be an ``OrcapodConfig`` instance. - - Returns: - New ``OrcapodConfig`` with merged sections. - - Raises: - TypeError: If ``other`` is not an ``OrcapodConfig`` instance. - """ - if not isinstance(other, OrcapodConfig): - raise TypeError("Can only merge with another OrcapodConfig instance") - return OrcapodConfig( - hashing=self.hashing.merge(other.hashing), - display=self.display.merge(other.display), - ) - - @classmethod - def from_dict( - cls, - data: dict[str, Any], - source_path: Path | str | None = None, - ) -> "OrcapodConfig": - """Construct an ``OrcapodConfig`` from a plain dict. - - Unknown top-level section names and unknown field names within a known - section are logged at WARNING level and otherwise ignored (forward-compat: - a config written by a newer orcapod will not break an older version). - - Args: - data: Mapping of section name to field dict (e.g. as produced by - ``dataclasses.asdict()`` or parsed from a TOML file). - source_path: Optional file path included in warning messages to help - users locate typos. - - Returns: - ``OrcapodConfig`` populated from ``data``; missing sections and fields - fall back to built-in defaults. - """ - known_sections = {"hashing", "display"} - path_str = f" in {source_path}" if source_path is not None else "" - - for key in data: - if key not in known_sections: - logger.warning( - "Unknown config section %r%s — ignored", key, path_str - ) - - hashing_dict = data.get("hashing", {}) - known_hashing = set(HashingConfig.__dataclass_fields__) - for key in hashing_dict: - if key not in known_hashing: - logger.warning( - "Unknown field %r in [hashing]%s — ignored", key, path_str - ) - hashing = HashingConfig( - **{k: v for k, v in hashing_dict.items() if k in known_hashing} - ) - - display_dict = data.get("display", {}) - known_display = set(DisplayConfig.__dataclass_fields__) - for key in display_dict: - if key not in known_display: - logger.warning( - "Unknown field %r in [display]%s — ignored", key, path_str - ) - display = DisplayConfig( - **{k: v for k, v in display_dict.items() if k in known_display} - ) - - return cls(hashing=hashing, display=display) - - -# Module-level default config — created at import time. -DEFAULT_CONFIG = OrcapodConfig() - - -def load_config( - project_config_path: Path | str | None = None, - user_config_path: Path | str | None = None, -) -> OrcapodConfig: - """Load and merge config from TOML files with precedence. - - Precedence (lowest to highest): - built-in defaults - → user-global config (``~/.orcapod/config.toml``) - → project-local config (``./orcapod_config.toml`` in cwd) - - Missing files are silently skipped. Malformed TOML raises ``ValueError`` - with the offending file path included in the message. - - Args: - project_config_path: Override the project-local config file path. - Defaults to ``orcapod_config.toml`` in the current working directory. - user_config_path: Override the user-global config file path. - Defaults to ``~/.orcapod/config.toml``. - - Returns: - Merged ``OrcapodConfig`` with all applicable overrides applied. - - Raises: - ValueError: If a config file exists but contains invalid TOML. - """ - import tomllib - - _user_path = ( - Path(user_config_path) - if user_config_path is not None - else Path.home() / ".orcapod" / "config.toml" - ) - _project_path = ( - Path(project_config_path) - if project_config_path is not None - else Path.cwd() / "orcapod_config.toml" - ) - - config = DEFAULT_CONFIG - - for path in (_user_path, _project_path): - if not path.exists(): - continue - try: - with open(path, "rb") as f: - data = tomllib.load(f) - except tomllib.TOMLDecodeError as e: - raise ValueError(f"Malformed TOML in {path}: {e}") from e - overlay = OrcapodConfig.from_dict(data, source_path=path) - config = config.merge(overlay) - - return config -``` - -- [ ] **Step 4: Run the new config tests** - -```bash -uv run pytest tests/test_orcapod_config.py -v -``` - -Expected: All tests in `TestHashingConfig`, `TestDisplayConfig`, `TestOrcapodConfig`, `TestOrcapodConfigFromDict`, `TestLoadConfig` PASS. `TestOrcapodConfigTopLevelExport` tests that import from `orcapod` will FAIL (exports not updated yet — fixed in Task 3). - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/config.py tests/test_orcapod_config.py -git commit -m "feat(config): restructure OrcapodConfig into nested HashingConfig/DisplayConfig sections; add from_dict() and load_config()" -``` - ---- - -## Task 2: Update all consumer files to new field paths - -**Files:** -- Modify: `src/orcapod/core/sources/stream_builder.py` -- Modify: `src/orcapod/core/nodes/source_node.py` -- Modify: `src/orcapod/core/operators/join.py` -- Modify: `src/orcapod/core/operators/merge_join.py` -- Modify: `src/orcapod/core/nodes/function_node.py` -- Modify: `src/orcapod/core/nodes/operator_node.py` -- Modify: `src/orcapod/utils/schema_utils.py` -- Modify: `tests/test_core/operators/test_operators.py` -- Modify: `tests/test_core/operators/test_merge_join.py` -- Modify: `tests/test_utils/test_schema_utils.py` - -This task is mechanical — no new tests. The existing tests for these files are the regression suite. - -- [ ] **Step 1: Record baseline failures from old field names** - -```bash -uv run pytest tests/ -x -q 2>&1 | head -30 -``` - -Expected: Multiple failures referencing `AttributeError: 'HashingConfig' object has no attribute 'system_tag_hash_n_char'` and similar. - -- [ ] **Step 2: Update `stream_builder.py` (2 changes)** - -In `src/orcapod/core/sources/stream_builder.py`: - -Line 134 — change: -```python - self._config.schema_hash_n_char, -``` -to: -```python - self._config.hashing.schema_n_char, -``` - -Line 142 — change: -```python - source_id = table_hash.to_hex(char_count=self._config.path_hash_n_char) -``` -to: -```python - source_id = table_hash.to_hex(char_count=self._config.hashing.path_n_char) -``` - -- [ ] **Step 3: Update `source_node.py` (1 change)** - -In `src/orcapod/core/nodes/source_node.py`: - -Line 130 — change: -```python - self.orcapod_config.schema_hash_n_char, -``` -to: -```python - self.orcapod_config.hashing.schema_n_char, -``` - -- [ ] **Step 4: Update `join.py` (3 changes)** - -In `src/orcapod/core/operators/join.py`, change all three occurrences of: -```python - n_char = self.orcapod_config.system_tag_hash_n_char -``` -to: -```python - n_char = self.orcapod_config.hashing.system_tag_n_char -``` - -Lines 98, 127, and 241. - -- [ ] **Step 5: Update `merge_join.py` (2 changes)** - -In `src/orcapod/core/operators/merge_join.py`, change both occurrences of: -```python - n_char = self.orcapod_config.system_tag_hash_n_char -``` -to: -```python - n_char = self.orcapod_config.hashing.system_tag_n_char -``` - -Lines 140 and 159. - -- [ ] **Step 6: Update `function_node.py` deserialization (2 changes)** - -In `src/orcapod/core/nodes/function_node.py`, change both occurrences of: -```python - OrcapodConfig(**config_dict) if config_dict is not None else DEFAULT_CONFIG -``` -to: -```python - OrcapodConfig.from_dict(config_dict) if config_dict is not None else DEFAULT_CONFIG -``` - -Lines 518 and 838. - -- [ ] **Step 7: Update `operator_node.py` deserialization (2 changes)** - -In `src/orcapod/core/nodes/operator_node.py`, change both occurrences of: -```python - OrcapodConfig(**config_dict) if config_dict is not None else DEFAULT_CONFIG -``` -to: -```python - OrcapodConfig.from_dict(config_dict) if config_dict is not None else DEFAULT_CONFIG -``` - -Lines 420 and 692. - -- [ ] **Step 8: Update `schema_utils.py` docstring (1 change)** - -In `src/orcapod/utils/schema_utils.py`, line 404 — change: -```python - (``OrcapodConfig.schema_hash_n_char``). -``` -to: -```python - (``OrcapodConfig.hashing.schema_n_char``). -``` - -- [ ] **Step 9: Update `test_operators.py` (2 changes)** - -In `tests/test_core/operators/test_operators.py`, change both occurrences of: -```python - n_char = OrcapodConfig().system_tag_hash_n_char -``` -to: -```python - n_char = OrcapodConfig().hashing.system_tag_n_char -``` - -Lines 1320 and 1415. - -- [ ] **Step 10: Update `test_merge_join.py` (1 change)** - -In `tests/test_core/operators/test_merge_join.py`, line 647 — change: -```python - n_char = OrcapodConfig().system_tag_hash_n_char -``` -to: -```python - n_char = OrcapodConfig().hashing.system_tag_n_char -``` - -- [ ] **Step 11: Update `test_schema_utils.py` (1 change)** - -In `tests/test_utils/test_schema_utils.py`, line 19 — change: -```python - return DEFAULT_CONFIG.schema_hash_n_char -``` -to: -```python - return DEFAULT_CONFIG.hashing.schema_n_char -``` - -- [ ] **Step 12: Run the full test suite to verify all passes** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: All tests PASS. Zero failures. - -- [ ] **Step 13: Commit** - -```bash -git add \ - src/orcapod/core/sources/stream_builder.py \ - src/orcapod/core/nodes/source_node.py \ - src/orcapod/core/operators/join.py \ - src/orcapod/core/operators/merge_join.py \ - src/orcapod/core/nodes/function_node.py \ - src/orcapod/core/nodes/operator_node.py \ - src/orcapod/utils/schema_utils.py \ - tests/test_core/operators/test_operators.py \ - tests/test_core/operators/test_merge_join.py \ - tests/test_utils/test_schema_utils.py -git commit -m "refactor(config): update all consumers to nested config field paths" -``` - ---- - -## Task 3: Update public API exports in `__init__.py` - -**Files:** -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Run the export tests to confirm they fail** - -```bash -uv run pytest tests/test_orcapod_config.py::TestOrcapodConfigTopLevelExport -v -``` - -Expected: All 8 tests FAIL — `HashingConfig`, `DisplayConfig`, `load_config` not yet exported. - -- [ ] **Step 2: Update `src/orcapod/__init__.py`** - -Replace the file with: - -```python -from .config import DEFAULT_CONFIG, DisplayConfig, HashingConfig, OrcapodConfig, load_config -from .core.function_pod import ( - FunctionPod, - function_pod, -) -from .core.nodes.source_node import SourceNode -from .pipeline import Pipeline, PipelineJob -from .semantic_types.dataclass_encoding import register_dataclass - -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -__all__ = [ - "DEFAULT_CONFIG", - "DisplayConfig", - "HashingConfig", - "OrcapodConfig", - "load_config", - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceNode", - "register_dataclass", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", -] -``` - -- [ ] **Step 3: Run the export tests** - -```bash -uv run pytest tests/test_orcapod_config.py::TestOrcapodConfigTopLevelExport -v -``` - -Expected: All 8 tests PASS. - -- [ ] **Step 4: Run the full test suite** - -```bash -uv run pytest tests/ -q -``` - -Expected: All tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/__init__.py -git commit -m "feat(orcapod): export HashingConfig, DisplayConfig, load_config from top-level package" -``` - ---- - -## Task 4: Full regression run and PR - -- [ ] **Step 1: Run the complete test suite one final time** - -```bash -uv run pytest tests/ -v -``` - -Expected: All tests PASS, zero failures, zero errors. - -- [ ] **Step 2: Draft the deferred save/load Linear issue** - -Create a new issue in Linear (team: Platform, project: Orcapod Python v0.1 Stabilization Push) titled **"Add save/write support to OrcaPod config system"** with description: - -``` -## Overview - -``load_config()`` (added in PLT-964) supports reading TOML config files, but there -is no way to programmatically write or update a config file. This issue adds -``save_config(config, path)`` so that config values set programmatically can be -persisted to disk. - -## Goals & Success Criteria - -* ``save_config(config: OrcapodConfig, path: Path | str)`` writes a TOML file -* Round-trip guarantee: ``load_config(project_config_path=p)`` after - ``save_config(cfg, p)`` returns a config equal to ``cfg`` -* Only non-default values are written (sparse output — default fields omitted) - -## Scope & Boundaries - -In scope: -* ``save_config()`` function in ``config.py`` -* Export from ``orcapod.__init__`` -* Tests for round-trip and sparse output - -Out of scope: -* CLI interface for config management -* Wiring display config into stream output (tracked in PLT-1570) - -## Dependencies & Risks - -* Depends on PLT-964 landing first -* Requires ``tomlkit`` or similar for round-trip-safe TOML writing (``tomllib`` - is read-only). New dependency needed. -``` - -- [ ] **Step 3: Push the branch** - -```bash -git push -u origin eywalker/plt-964-add-persistent-orcapod-configuration-system -``` - -- [ ] **Step 4: Open the PR** - -```bash -gh pr create \ - --base dev \ - --title "feat(config): add persistent OrcaPod configuration system (PLT-964)" \ - --body "$(cat <<'EOF' -## Summary - -- Restructures `OrcapodConfig` into a nested sectioned design (`HashingConfig` + `DisplayConfig`) -- Renames flat hashing fields to drop redundant `_hash_` infix (e.g. `system_tag_hash_n_char` → `hashing.system_tag_n_char`) -- Adds `OrcapodConfig.from_dict()` — fixes the broken `OrcapodConfig(**config_dict)` deserialization pattern in `function_node.py` and `operator_node.py` -- Adds `load_config()` — reads from `~/.orcapod/config.toml` (user-global) and `./orcapod_config.toml` (project-local) with precedence; uses built-in `tomllib`, no new deps -- Unknown config keys emit `logging.WARNING` (forward-compat: newer config files don't break older library versions) -- Exports `HashingConfig`, `DisplayConfig`, `load_config` from top-level `orcapod` package - -Closes PLT-964 - -## Test plan - -- [ ] `uv run pytest tests/test_orcapod_config.py -v` — all new structural, `from_dict`, and `load_config` tests pass -- [ ] `uv run pytest tests/ -q` — full suite passes, no regressions - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` diff --git a/superpowers/plans/2026-06-11-plt-1162-uuid-arrow-type.md b/superpowers/plans/2026-06-11-plt-1162-uuid-arrow-type.md deleted file mode 100644 index 8884082e8..000000000 --- a/superpowers/plans/2026-06-11-plt-1162-uuid-arrow-type.md +++ /dev/null @@ -1,956 +0,0 @@ -# UUID Arrow Type Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace all `pa.large_string()` UUID storage in OrcaPod with `pa.binary(16)` (`fixed_size_binary[16]`), introducing two named constants and a semantic struct converter. - -**Architecture:** A single `UUID_ARROW_TYPE = pa.binary(16)` constant in `types.py` is used for all system UUID columns; a `UUID_STRUCT_ARROW_TYPE = pa.struct([pa.field("uuid", pa.binary(16))])` constant enables `uuid.UUID` Python objects to round-trip through the semantic type system. All UUID generation sites switch from `str(uuid7())` to `uuid7().bytes`. - -**Tech Stack:** PyArrow 23, `uuid_utils` (uuid7), `uuid` stdlib, pytest - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/types.py` | Modify | Add `UUID_ARROW_TYPE`, `UUID_STRUCT_ARROW_TYPE` constants | -| `src/orcapod/__init__.py` | Modify | Export both new constants | -| `src/orcapod/semantic_types/semantic_struct_converters.py` | Modify | Add `UUIDStructConverter` class | -| `src/orcapod/hashing/versioned_hashers.py` | Modify | Register `UUIDStructConverter` in default registry | -| `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Modify | `UUIDHandler.handle` returns `obj.bytes` not `str(obj)` | -| `src/orcapod/utils/arrow_data_utils.py` | Modify | System UUID column field types → `UUID_ARROW_TYPE` | -| `src/orcapod/core/datagrams/datagram.py` | Modify | `str(uuid7())` → `uuid7().bytes`; `_datagram_id` type `bytes` | -| `src/orcapod/core/data_function.py` | Modify | `str(uuid7())` → `uuid7().bytes`; `record_id` type `bytes` | -| `src/orcapod/pipeline/logging_observer.py` | Modify | `str(uuid7())` → `uuid7().bytes` | -| `src/orcapod/pipeline/status_observer.py` | Modify | `str(uuid7())` → `uuid7().bytes` | -| `src/orcapod/databases/postgresql_connector.py` | Modify | `uuid` pg type → `UUID_ARROW_TYPE`; decode driver values to bytes | -| `tests/test_types.py` | Modify | Add constant assertions | -| `tests/test_semantic_types/test_uuid_struct_converter.py` | Create | Full converter test suite | -| `tests/test_semantic_types/test_semantic_registry.py` | Modify | Assert UUID type registered | -| `tests/` (various) | Modify | Fix tests asserting `pa.large_string()` for UUID columns | - ---- - -### Task 1: UUID Arrow type constants - -**Files:** -- Modify: `src/orcapod/types.py` -- Modify: `src/orcapod/__init__.py` -- Modify: `tests/test_types.py` - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_types.py`: - -```python -import pyarrow as pa - -from orcapod.types import UUID_ARROW_TYPE, UUID_STRUCT_ARROW_TYPE - - -def test_uuid_arrow_type_is_binary16(): - assert UUID_ARROW_TYPE == pa.binary(16) - - -def test_uuid_struct_arrow_type_structure(): - assert UUID_STRUCT_ARROW_TYPE == pa.struct([pa.field("uuid", pa.binary(16))]) - - -def test_uuid_struct_inner_type_matches_constant(): - assert UUID_STRUCT_ARROW_TYPE.field("uuid").type == UUID_ARROW_TYPE -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_types.py::test_uuid_arrow_type_is_binary16 tests/test_types.py::test_uuid_struct_arrow_type_structure -v -``` - -Expected: `ImportError: cannot import name 'UUID_ARROW_TYPE'` - -- [ ] **Step 3: Add constants to `types.py`** - -Find the import block at the top of `src/orcapod/types.py`. After the `if TYPE_CHECKING:` block (around line 22), add: - -```python -if TYPE_CHECKING: - import pyarrow as pa - # ... existing ... -else: - from orcapod.utils.lazy_module import LazyModule - pa = LazyModule("pyarrow") -``` - -Then at module level, after the imports, before the first class definition, add: - -```python -# --------------------------------------------------------------------------- -# UUID Arrow type constants -# --------------------------------------------------------------------------- - -def _make_uuid_types() -> "tuple[pa.DataType, pa.StructType]": - """Build UUID Arrow types (deferred so pyarrow is not imported at module level).""" - import pyarrow as _pa - arrow_type = _pa.binary(16) - struct_type = _pa.struct([_pa.field("uuid", arrow_type)]) - return arrow_type, struct_type - - -# Canonical Arrow type for all UUID values in OrcaPod. -# Stored as fixed_size_binary[16] — 16 raw bytes, no hex encoding, no dashes. -UUID_ARROW_TYPE: "pa.DataType" -# Semantic struct type for Python uuid.UUID round-trips through the type system. -UUID_STRUCT_ARROW_TYPE: "pa.StructType" -UUID_ARROW_TYPE, UUID_STRUCT_ARROW_TYPE = _make_uuid_types() -del _make_uuid_types -``` - -> **Note on lazy import:** The rest of `types.py` uses `LazyModule("pyarrow")` to defer the heavy import. For module-level constants that need the actual `pa` object at definition time, the pattern is to call a helper function that imports `pyarrow` directly. The assignment happens at module import time but is deferred inside the helper. - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_types.py::test_uuid_arrow_type_is_binary16 tests/test_types.py::test_uuid_struct_arrow_type_structure tests/test_types.py::test_uuid_struct_inner_type_matches_constant -v -``` - -Expected: all PASS - -- [ ] **Step 5: Export from `src/orcapod/__init__.py`** - -Open `src/orcapod/__init__.py` and find the existing imports from `types`. Add `UUID_ARROW_TYPE` and `UUID_STRUCT_ARROW_TYPE` to the same import line: - -```python -from orcapod.types import ( - # ... existing exports ... - UUID_ARROW_TYPE, - UUID_STRUCT_ARROW_TYPE, -) -``` - -- [ ] **Step 6: Verify import from top-level package** - -```bash -uv run python -c "from orcapod import UUID_ARROW_TYPE, UUID_STRUCT_ARROW_TYPE; import pyarrow as pa; assert UUID_ARROW_TYPE == pa.binary(16); print('OK')" -``` - -Expected: `OK` - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/types.py src/orcapod/__init__.py tests/test_types.py -git commit -m "feat(types): add UUID_ARROW_TYPE and UUID_STRUCT_ARROW_TYPE constants (PLT-1162)" -``` - ---- - -### Task 2: UUIDStructConverter - -**Files:** -- Modify: `src/orcapod/semantic_types/semantic_struct_converters.py` -- Create: `tests/test_semantic_types/test_uuid_struct_converter.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_semantic_types/test_uuid_struct_converter.py`: - -```python -"""Tests for UUIDStructConverter.""" -import uuid - -import pyarrow as pa -import pytest - -from orcapod.semantic_types.semantic_struct_converters import UUIDStructConverter -from orcapod.types import UUID_ARROW_TYPE, UUID_STRUCT_ARROW_TYPE - - -@pytest.fixture -def converter(): - return UUIDStructConverter() - - -@pytest.fixture -def sample_uuid(): - return uuid.UUID("550e8400-e29b-41d4-a716-446655440000") - - -def test_python_type(converter): - assert converter.python_type is uuid.UUID - - -def test_arrow_struct_type(converter): - assert converter.arrow_struct_type == UUID_STRUCT_ARROW_TYPE - - -def test_semantic_type_name(converter): - assert converter.semantic_type_name == "uuid" - - -def test_python_to_struct_dict(converter, sample_uuid): - result = converter.python_to_struct_dict(sample_uuid) - assert result == {"uuid": sample_uuid.bytes} - assert isinstance(result["uuid"], bytes) - assert len(result["uuid"]) == 16 - - -def test_python_to_struct_dict_rejects_non_uuid(converter): - with pytest.raises(TypeError): - converter.python_to_struct_dict("550e8400-e29b-41d4-a716-446655440000") # type: ignore - - -def test_struct_dict_to_python(converter, sample_uuid): - struct_dict = {"uuid": sample_uuid.bytes} - result = converter.struct_dict_to_python(struct_dict) - assert result == sample_uuid - assert isinstance(result, uuid.UUID) - - -def test_struct_dict_to_python_from_bytearray(converter, sample_uuid): - """Arrow may return binary fields as bytearray — must handle both.""" - struct_dict = {"uuid": bytearray(sample_uuid.bytes)} - result = converter.struct_dict_to_python(struct_dict) - assert result == sample_uuid - - -def test_struct_dict_to_python_missing_field(converter): - with pytest.raises(ValueError, match="Missing 'uuid' field"): - converter.struct_dict_to_python({}) - - -def test_round_trip(converter, sample_uuid): - struct_dict = converter.python_to_struct_dict(sample_uuid) - recovered = converter.struct_dict_to_python(struct_dict) - assert recovered == sample_uuid - - -def test_round_trip_all_versions(): - """Verify round-trip works for uuid4, uuid5, and uuid7 (uuid_utils).""" - from uuid_utils import uuid7 - - converter = UUIDStructConverter() - for u in [uuid.uuid4(), uuid.uuid5(uuid.NAMESPACE_OID, "test"), uuid7()]: - assert converter.struct_dict_to_python(converter.python_to_struct_dict(u)) == u - - -def test_arrow_array_round_trip(converter, sample_uuid): - """Verify UUID survives a PyArrow array round-trip.""" - struct_dict = converter.python_to_struct_dict(sample_uuid) - arr = pa.array([struct_dict], type=UUID_STRUCT_ARROW_TYPE) - recovered_dict = arr[0].as_py() - recovered_uuid = converter.struct_dict_to_python(recovered_dict) - assert recovered_uuid == sample_uuid - - -def test_distinct_uuids_produce_distinct_struct_dicts(converter): - u1, u2 = uuid.uuid4(), uuid.uuid4() - assert converter.python_to_struct_dict(u1) != converter.python_to_struct_dict(u2) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_uuid_struct_converter.py -v -``` - -Expected: `ImportError: cannot import name 'UUIDStructConverter'` - -- [ ] **Step 3: Add `UUIDStructConverter` to `semantic_struct_converters.py`** - -Add the following imports at the top of `src/orcapod/semantic_types/semantic_struct_converters.py` (after existing imports): - -```python -import uuid as _uuid_module -from typing import Any - -from orcapod.types import UUID_STRUCT_ARROW_TYPE -``` - -Then add the class after `UPathStructConverter`: - -```python -class UUIDStructConverter(SemanticStructConverterBase): - """Converts Python ``uuid.UUID`` objects to/from the OrcaPod UUID struct. - - Stores UUIDs as their raw 16-byte binary representation inside a - single-field struct ``struct``. This follows - the same single-field struct pattern used by ``PythonPathStructConverter`` - and ``UPathStructConverter``. - """ - - def __init__(self) -> None: - super().__init__("uuid") - self._arrow_struct_type = UUID_STRUCT_ARROW_TYPE - - @property - def python_type(self) -> type: - """Python type handled by this converter.""" - return _uuid_module.UUID - - @property - def arrow_struct_type(self) -> "pa.StructType": - """Arrow struct type for UUID values.""" - return self._arrow_struct_type - - def python_to_struct_dict(self, value: Any) -> dict[str, bytes]: - """Convert a ``uuid.UUID`` to a struct dictionary. - - Args: - value: A ``uuid.UUID`` instance. - - Returns: - ``{"uuid": <16 bytes>}`` - - Raises: - TypeError: If ``value`` is not a ``uuid.UUID``. - """ - if not isinstance(value, _uuid_module.UUID): - raise TypeError( - f"Expected uuid.UUID, got {type(value).__name__}" - ) - return {"uuid": value.bytes} - - def struct_dict_to_python(self, struct_dict: dict[str, Any]) -> _uuid_module.UUID: - """Convert a struct dictionary back to a ``uuid.UUID``. - - Args: - struct_dict: Dict with a ``"uuid"`` key holding 16 bytes. - - Returns: - The reconstructed ``uuid.UUID``. - - Raises: - ValueError: If the ``"uuid"`` field is missing. - """ - raw = struct_dict.get("uuid") - if raw is None: - raise ValueError("Missing 'uuid' field in struct dict") - return _uuid_module.UUID(bytes=bytes(raw)) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_uuid_struct_converter.py -v -``` - -Expected: all PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/semantic_struct_converters.py \ - tests/test_semantic_types/test_uuid_struct_converter.py -git commit -m "feat(semantic-types): add UUIDStructConverter for uuid.UUID Arrow struct round-trips (PLT-1162)" -``` - ---- - -### Task 3: Register UUIDStructConverter in the default semantic registry - -**Files:** -- Modify: `src/orcapod/hashing/versioned_hashers.py` -- Modify: `tests/test_semantic_types/test_semantic_registry.py` - -- [ ] **Step 1: Find the fixture name used in existing registry tests** - -```bash -grep -n "def.*registry\|fixture.*registry" \ - tests/test_semantic_types/test_semantic_registry.py tests/conftest.py 2>/dev/null | head -10 -``` - -Note the fixture name (e.g. `default_registry`, `registry`, `semantic_registry`). Use that exact name in the new tests below. - -- [ ] **Step 1b: Write the failing tests** - -Open `tests/test_semantic_types/test_semantic_registry.py` and add (replacing `default_registry` with the fixture name found above): - -```python -import uuid - -import pyarrow as pa - -from orcapod.types import UUID_STRUCT_ARROW_TYPE - - -def test_uuid_type_registered_in_default_registry(default_registry): - """uuid.UUID should be registered and map to UUID_STRUCT_ARROW_TYPE.""" - struct_type = default_registry.get_struct_for_python_type(uuid.UUID) - assert struct_type == UUID_STRUCT_ARROW_TYPE - - -def test_uuid_struct_resolves_to_python_type(default_registry): - """UUID_STRUCT_ARROW_TYPE should resolve back to uuid.UUID.""" - python_type = default_registry.get_python_type_for_struct(UUID_STRUCT_ARROW_TYPE) - assert python_type is uuid.UUID - - -def test_uuid_semantic_type_name_registered(default_registry): - """Converter registered under the name 'uuid'.""" - converter = default_registry.get_converter_for_semantic_type("uuid") - assert converter is not None - assert converter.python_type is uuid.UUID -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_semantic_registry.py::test_uuid_type_registered_in_default_registry -v -``` - -Expected: FAIL — `uuid.UUID` not registered - -- [ ] **Step 3: Register the converter in `versioned_hashers.py`** - -Open `src/orcapod/hashing/versioned_hashers.py`. Find the block that registers `PythonPathStructConverter` (around line 125-138): - -```python - from orcapod.semantic_types.semantic_struct_converters import PythonPathStructConverter - - registry: Any = SemanticTypeRegistry() - file_hasher = BasicFileHasher(algorithm="sha256") - path_converter: Any = PythonPathStructConverter(file_hasher=file_hasher) - registry.register_converter("path", path_converter) -``` - -Extend it to also register `UUIDStructConverter`: - -```python - from orcapod.semantic_types.semantic_struct_converters import ( - PythonPathStructConverter, - UUIDStructConverter, - ) - - registry: Any = SemanticTypeRegistry() - file_hasher = BasicFileHasher(algorithm="sha256") - path_converter: Any = PythonPathStructConverter(file_hasher=file_hasher) - registry.register_converter("path", path_converter) - uuid_converter: Any = UUIDStructConverter() - registry.register_converter("uuid", uuid_converter) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_semantic_registry.py::test_uuid_type_registered_in_default_registry tests/test_semantic_types/test_semantic_registry.py::test_uuid_struct_resolves_to_python_type tests/test_semantic_types/test_semantic_registry.py::test_uuid_semantic_type_name_registered -v -``` - -Expected: all PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/hashing/versioned_hashers.py \ - tests/test_semantic_types/test_semantic_registry.py -git commit -m "feat(semantic-types): register UUIDStructConverter in default semantic registry (PLT-1162)" -``` - ---- - -### Task 4: Update UUIDHandler in semantic hashing - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Modify: (existing test file for builtin handlers — find with `grep -rl "UUIDHandler" tests/`) - -- [ ] **Step 1: Find the existing UUIDHandler test** - -```bash -grep -rl "UUIDHandler\|uuid.*handler\|handler.*uuid" tests/ --include="*.py" -i -``` - -If a file is found, open it and locate the test asserting that handling a `uuid.UUID` returns its string form — update it per Step 2 below. - -If **no file is found**, create `tests/test_hashing/test_uuid_handler.py`: - -```python -"""Tests for UUIDHandler in semantic hashing.""" -``` - -Then proceed to Step 2. - -- [ ] **Step 2: Update the test to expect bytes** - -In the test file found above, update the UUID handler test: - -```python -import uuid as _uuid - -def test_uuid_handler_returns_bytes(): - """UUIDHandler should return the 16-byte binary representation.""" - from orcapod.hashing.semantic_hashing.builtin_handlers import UUIDHandler - - handler = UUIDHandler() - u = _uuid.UUID("550e8400-e29b-41d4-a716-446655440000") - result = handler.handle(u, hasher=None) # type: ignore[arg-type] - assert result == u.bytes - assert isinstance(result, bytes) - assert len(result) == 16 - - -def test_uuid_handler_different_uuids_produce_different_bytes(): - from orcapod.hashing.semantic_hashing.builtin_handlers import UUIDHandler - - handler = UUIDHandler() - u1 = _uuid.uuid4() - u2 = _uuid.uuid4() - assert handler.handle(u1, None) != handler.handle(u2, None) # type: ignore[arg-type] -``` - -- [ ] **Step 3: Run to verify test fails** - -```bash -uv run pytest tests/ -k "uuid_handler" -v -``` - -Expected: FAIL — result is a string, not bytes - -- [ ] **Step 4: Update `UUIDHandler` in `builtin_handlers.py`** - -Open `src/orcapod/hashing/semantic_hashing/builtin_handlers.py`. Find `UUIDHandler` (around line 141): - -```python -class UUIDHandler: - """ - Handler for uuid.UUID objects. - - Converts the UUID to its canonical hyphenated string representation - (e.g. ``"550e8400-e29b-41d4-a716-446655440000"``), which is stable, - human-readable, and unambiguous. - """ - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: - return str(obj) -``` - -Replace with: - -```python -class UUIDHandler: - """Handler for ``uuid.UUID`` objects. - - Returns the raw 16-byte binary representation of the UUID, consistent - with OrcaPod's canonical ``pa.binary(16)`` Arrow storage format. - The binary form is compact, unambiguous, and independent of string - formatting conventions. - """ - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: - return obj.bytes -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -uv run pytest tests/ -k "uuid_handler" -v -``` - -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py -git commit -m "fix(hashing): UUIDHandler returns bytes instead of str, consistent with binary Arrow storage (PLT-1162)" -``` - ---- - -### Task 5: Update system UUID column Arrow types in arrow_data_utils.py - -**Files:** -- Modify: `src/orcapod/utils/arrow_data_utils.py` - -- [ ] **Step 1: Locate all UUID column field definitions** - -The UUID columns that need updating are **only** the following — all others (e.g. `source_id`, `pipeline_hash`, string metadata) are NOT UUIDs and must be left as `pa.large_string()`: - -| Column name | File | Contains UUID? | -|---|---|---| -| `datagram_id` | `datagram.py`, `arrow_data_utils.py` | ✅ uuid7 | -| `record_id` / `record_id_col_name` | `arrow_data_utils.py`, `data_function.py` | ✅ uuid7 | -| `_log_id` | `logging_observer.py` | ✅ uuid7 | -| `_status_id` | `status_observer.py` | ✅ uuid7 | -| `source_id` | `arrow_data_utils.py`, `stream_builder.py` | ❌ hash string | -| `_source_*` columns | `arrow_data_utils.py` | ❌ provenance string | -| `pipeline_hash` | various | ❌ hash string | - -Run this to confirm which lines in `arrow_data_utils.py` define UUID-carrying fields: - -```bash -grep -n "record_id\|datagram_id\|_log_id\|_status_id" \ - src/orcapod/utils/arrow_data_utils.py | grep -i "field\|large_string\|array" -``` - -- [ ] **Step 2: Update imports in `arrow_data_utils.py`** - -Add the import for `UUID_ARROW_TYPE` at the top of `src/orcapod/utils/arrow_data_utils.py`, alongside any existing `orcapod` imports: - -```python -from orcapod.types import UUID_ARROW_TYPE -``` - -- [ ] **Step 3: Replace UUID column field types** - -For every `pa.field(record_id_col_name, pa.large_string(), ...)` and `pa.array(..., type=pa.large_string())` that is used for UUID columns (identified in Step 1), replace `pa.large_string()` with `UUID_ARROW_TYPE`. Leave non-UUID `large_string` columns (e.g. `source_id` hash columns, string metadata) unchanged. - -Example — if you see: - -```python -pa.field(record_id_col_name, pa.large_string(), nullable=False) -``` - -Change to: - -```python -pa.field(record_id_col_name, UUID_ARROW_TYPE, nullable=False) -``` - -And if you see an array constructor for record IDs: - -```python -pa.array(record_ids, type=pa.large_string()) -``` - -Change to: - -```python -pa.array(record_ids, type=UUID_ARROW_TYPE) -``` - -- [ ] **Step 4: Run the relevant test suite to surface failures** - -```bash -uv run pytest tests/test_core/ tests/test_utils/ -x -v 2>&1 | head -60 -``` - -Fix any type assertion failures by updating tests that check `pa.large_string()` for UUID columns to check `pa.binary(16)` instead. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/utils/arrow_data_utils.py tests/ -git commit -m "fix(arrow): update system UUID column Arrow types to pa.binary(16) (PLT-1162)" -``` - ---- - -### Task 6: Update UUID generation sites to produce bytes - -**Files:** -- Modify: `src/orcapod/core/datagrams/datagram.py` -- Modify: `src/orcapod/core/data_function.py` -- Modify: `src/orcapod/pipeline/logging_observer.py` -- Modify: `src/orcapod/pipeline/status_observer.py` - -- [ ] **Step 1: Update `datagram.py`** - -Open `src/orcapod/core/datagrams/datagram.py`. Find the `datagram_id` property (around line 432): - -```python -@property -def datagram_id(self) -> str: - if self._datagram_id is None: - self._datagram_id = str(uuid7()) - return self._datagram_id -``` - -Change to: - -```python -@property -def datagram_id(self) -> bytes: - if self._datagram_id is None: - self._datagram_id = uuid7().bytes - return self._datagram_id -``` - -Also update the `_datagram_id` instance variable type annotation (if present) from `str | None` to `bytes | None`, and update the `__init__` parameter type annotation for `datagram_id` from `str | None` to `bytes | None`. - -- [ ] **Step 2: Update `data_function.py`** - -Open `src/orcapod/core/data_function.py`. Find the `record_id` generation (around line 555): - -```python -record_id = str(uuid7()) -``` - -Change to: - -```python -record_id = uuid7().bytes -``` - -Update any type annotation for `record_id` in the same method from `str` to `bytes`. - -- [ ] **Step 3: Update `logging_observer.py`** - -Open `src/orcapod/pipeline/logging_observer.py`. Find the `_log_id` generation (around line 103): - -```bash -grep -n "uuid7\|log_id" src/orcapod/pipeline/logging_observer.py -``` - -Change all `str(uuid7())` used for Arrow-stored IDs to `uuid7().bytes`. - -- [ ] **Step 4: Update `status_observer.py`** - -Open `src/orcapod/pipeline/status_observer.py`. Find the `_status_id` generation (lines 273 and 400): - -```bash -grep -n "uuid7\|status_id" src/orcapod/pipeline/status_observer.py -``` - -Change all `str(uuid7())` used for Arrow-stored IDs to `uuid7().bytes`. - -- [ ] **Step 5: Run the core test suite** - -```bash -uv run pytest tests/test_core/datagrams/ tests/test_core/data_function/ \ - tests/test_observability/ -x -v 2>&1 | head -80 -``` - -Fix any type assertion failures — tests that previously checked `isinstance(datagram_id, str)` should now check `isinstance(datagram_id, bytes)` and `len(datagram_id) == 16`. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/datagrams/datagram.py \ - src/orcapod/core/data_function.py \ - src/orcapod/pipeline/logging_observer.py \ - src/orcapod/pipeline/status_observer.py \ - tests/ -git commit -m "fix(core): UUID generation sites produce bytes instead of str (PLT-1162)" -``` - ---- - -### Task 7: Update PostgreSQL connector UUID mapping - -**Files:** -- Modify: `src/orcapod/databases/postgresql_connector.py` -- Modify: `tests/test_databases/` (find PostgreSQL connector test) - -- [ ] **Step 1: Locate and write the failing test** - -```bash -grep -rl "PostgreSQL\|postgresql\|pg_type_to_arrow\|uuid" tests/test_databases/ --include="*.py" | head -5 -``` - -Open the relevant test file and add: - -```python -import pyarrow as pa -from orcapod.types import UUID_ARROW_TYPE - - -def test_pg_uuid_maps_to_binary16(): - """PostgreSQL 'uuid' type must map to UUID_ARROW_TYPE = pa.binary(16).""" - from orcapod.databases.postgresql_connector import _pg_type_to_arrow - - result = _pg_type_to_arrow("uuid") - assert result == UUID_ARROW_TYPE - assert result == pa.binary(16) -``` - -- [ ] **Step 2: Run to verify test fails** - -```bash -uv run pytest tests/test_databases/ -k "pg_uuid_maps" -v -``` - -Expected: FAIL — returns `pa.large_string()` - -- [ ] **Step 3: Update `_pg_type_to_arrow` in `postgresql_connector.py`** - -Open `src/orcapod/databases/postgresql_connector.py`. Add the import near the top of the file (or inside the function): - -```python -from orcapod.types import UUID_ARROW_TYPE -``` - -Find the UUID branch in `_pg_type_to_arrow` (around line 95-100): - -```python - if t in ("text", "varchar", "character varying", "char", "bpchar", - "name", "uuid", "json", "jsonb", "time", "timetz"): - if t in ("time", "timetz"): - logger.warning("PostgreSQL type %r mapped to pa.large_string() (known gap)", t) - if t == "uuid": - # TODO: revisit mapping once PLT-1162 decides on a canonical UUID Arrow type - pass - return _pa.large_string() -``` - -Restructure to handle `uuid` separately before the grouped string branch: - -```python - if t == "uuid": - return UUID_ARROW_TYPE - - if t in ("text", "varchar", "character varying", "char", "bpchar", - "name", "json", "jsonb", "time", "timetz"): - if t in ("time", "timetz"): - logger.warning("PostgreSQL type %r mapped to pa.large_string() (known gap)", t) - return _pa.large_string() -``` - -- [ ] **Step 4: Handle bytes conversion in array construction** - -The PostgreSQL driver (`psycopg2`) returns `uuid` column values as Python `uuid.UUID` objects. When building Arrow arrays from fetched rows the connector must convert these to `bytes` before passing to `pa.binary(16)`. - -First, locate the array-construction code: - -```bash -grep -n "pa\.array\|pa\.table\|pa\.record_batch\|pyarrow\|_coerce\|_build\|_rows_to" \ - src/orcapod/databases/postgresql_connector.py | head -30 -``` - -The connector will have a loop that collects column values and calls `pa.array(column_values, type=arrow_type)`. Add the following module-level helper above that loop: - -```python -def _coerce_pg_value(value: Any, arrow_type: "pa.DataType") -> Any: - """Coerce a psycopg2 value to match the target Arrow type. - - Args: - value: Raw value from the psycopg2 cursor. - arrow_type: The Arrow type the value will be stored as. - - Returns: - A Python value compatible with ``pa.array(..., type=arrow_type)``. - """ - import pyarrow as _pa - if value is None: - return None - # psycopg2 returns uuid columns as uuid.UUID objects; binary(16) needs bytes - if arrow_type == _pa.binary(16) and hasattr(value, "bytes"): - return value.bytes - return value -``` - -Then in the column-building loop, wrap each value: - -```python -# Before -column_values = [row[col_idx] for row in rows] - -# After -column_values = [_coerce_pg_value(row[col_idx], arrow_type) for row in rows] -``` - -If the connector uses a different pattern (e.g. a single `pa.Table.from_pydict` call), apply the same coercion on the dict values for any `pa.binary(16)` column. - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -uv run pytest tests/test_databases/ -x -v 2>&1 | head -60 -``` - -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/databases/postgresql_connector.py tests/test_databases/ -git commit -m "fix(databases): map PostgreSQL uuid columns to UUID_ARROW_TYPE (pa.binary(16)) (PLT-1162)" -``` - ---- - -### Task 8: Full test suite — fix remaining failures - -- [ ] **Step 1: Run the full test suite** - -```bash -uv run pytest tests/ -x --tb=short 2>&1 | tail -40 -``` - -- [ ] **Step 2: For each failure, apply the fix pattern** - -Most remaining failures will be one of these patterns: - -**Pattern A — test asserts `pa.large_string()` for a UUID column:** -```python -# Before (failing) -assert schema.field("datagram_id").type == pa.large_string() -# After (fixed) -assert schema.field("datagram_id").type == pa.binary(16) -``` - -**Pattern B — test constructs a UUID value as a string:** -```python -# Before (failing) -datagram = Datagram(datagram_id="550e8400-e29b-41d4-a716-446655440000") -# After (fixed) -import uuid -datagram = Datagram(datagram_id=uuid.UUID("550e8400-e29b-41d4-a716-446655440000").bytes) -``` - -**Pattern C — test checks `isinstance(x, str)` for a UUID column value:** -```python -# Before (failing) -assert isinstance(datagram.datagram_id, str) -# After (fixed) -assert isinstance(datagram.datagram_id, bytes) -assert len(datagram.datagram_id) == 16 -``` - -- [ ] **Step 3: Run the full suite again to confirm all pass** - -```bash -uv run pytest tests/ --tb=short 2>&1 | tail -20 -``` - -Expected: all tests PASS, no failures - -- [ ] **Step 4: Commit** - -```bash -git add tests/ -git commit -m "test: update UUID column type assertions from large_string to binary(16) (PLT-1162)" -``` - ---- - -### Task 9: Post-implementation linear comment - -- [ ] **Step 1: Post design note to PLT-1162** - -Post a comment on [PLT-1162](https://linear.app/enigma-metamorphic/issue/PLT-1162) summarising the decision: - -``` -mcp__claude_ai_Linear__save_comment( - issueId: "PLT-1162", - body: "## Decision - -Canonical UUID Arrow type: `pa.binary(16)` (`fixed_size_binary[16]`) for all UUID values in OrcaPod. - -**Two layers:** -- `UUID_ARROW_TYPE = pa.binary(16)` — all system UUID columns (datagram_id, record_id, _log_id, _status_id) -- `UUID_STRUCT_ARROW_TYPE = pa.struct([pa.field(\"uuid\", pa.binary(16))])` — semantic type for uuid.UUID Python round-trips - -**Alternatives rejected:** -- `pa.large_string()` — 2.75× larger, no type distinction from arbitrary strings -- `pa.uuid()` — Polars 1.31 hard incompatibility (ComputeError on conversion) -- struct with large_string inner type — UUID is fundamentally binary, not a string - -**Known limitation:** Polars `write_json()` panics on Binary columns (pola-rs/polars#15410). OrcaPod has zero `write_json()` calls; unaffected in practice. - -Full spec: `superpowers/specs/2026-06-11-plt-1162-uuid-arrow-type-design.md`" -) -``` - ---- - -### Task 10: Push and create PR - -- [ ] **Step 1: Push branch** - -```bash -git push -u origin eywalker/plt-1162-design-spike-uuid-arrow-type-mapping-large_string-vs -``` - -- [ ] **Step 2: Invoke `sensei:create-pr`** - -Use the `sensei:create-pr` skill to create the pull request targeting `main`. diff --git a/superpowers/plans/2026-06-11-type-specific-pod-access.md b/superpowers/plans/2026-06-11-type-specific-pod-access.md deleted file mode 100644 index d9d2d70c1..000000000 --- a/superpowers/plans/2026-06-11-type-specific-pod-access.md +++ /dev/null @@ -1,232 +0,0 @@ -# Type-Specific Pod Access Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `source_pods`, `function_pods`, and `operator_pods` read-only properties to `AbstractPipelineBase` and `PipelineProtocol`, each returning a filtered `dict[str, Any]` subset of `.nodes`. - -**Architecture:** Each property is a one-liner dict comprehension filtering `self._nodes` by `node_type`. The properties live on `AbstractPipelineBase` so both `Pipeline` and `PipelineJob` inherit them for free. `PipelineProtocol` gets matching stubs so protocol-typed callers can use the new accessors without downcasting. - -**Tech Stack:** Python, pytest, uv (all tests run via `uv run pytest`) - ---- - -## File Map - -| File | Change | -|---|---| -| `tests/test_pipeline/test_pipeline.py` | Add `TestTypePodAccess` class with 6 tests | -| `src/orcapod/pipeline/base.py` | Add 3 properties to `AbstractPipelineBase` after `.nodes` | -| `src/orcapod/protocols/pipeline_protocols.py` | Add 3 property stubs to `PipelineProtocol` after `.nodes` | - ---- - -## Task 1: Write failing tests - -**Files:** -- Modify: `tests/test_pipeline/test_pipeline.py` (append new class at end of file) - -- [ ] **Step 1: Append `TestTypePodAccess` to `tests/test_pipeline/test_pipeline.py`** - -Add the following class at the very end of the file (after the last existing class): - -```python -# --------------------------------------------------------------------------- -# Tests: Type-specific pod access (PLT-420) -# --------------------------------------------------------------------------- - - -class TestTypePodAccess: - """Tests for source_pods, function_pods, and operator_pods properties.""" - - def test_function_pods_returns_only_function_nodes(self): - src = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - pipeline = Pipeline(name="test") - with pipeline: - pod(src, label="doubled") - fp = pipeline.function_pods - assert len(fp) == 1 - assert "doubled" in fp - assert isinstance(fp["doubled"], FunctionNode) - - def test_source_pods_returns_only_source_nodes(self): - src = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - pipeline = Pipeline(name="test") - with pipeline: - pod(src, label="doubled") - sp = pipeline.source_pods - assert len(sp) == 1 - assert isinstance(list(sp.values())[0], SourceNode) - - def test_operator_pods_returns_only_operator_nodes(self): - src_a, src_b = _make_two_sources() - pipeline = Pipeline(name="test") - with pipeline: - Join()(src_a, src_b, label="joined") - op = pipeline.operator_pods - assert len(op) == 1 - assert "joined" in op - assert isinstance(op["joined"], OperatorNode) - - def test_type_pods_union_covers_all_nodes(self): - src_a, src_b = _make_two_sources() - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - pipeline = Pipeline(name="test") - with pipeline: - joined = Join()(src_a, src_b, label="joined") - pod(joined, label="doubled") - all_nodes = pipeline.nodes - combined = { - **pipeline.source_pods, - **pipeline.function_pods, - **pipeline.operator_pods, - } - assert combined == all_nodes - - def test_function_pods_empty_when_no_function_nodes(self): - src_a, src_b = _make_two_sources() - pipeline = Pipeline(name="test") - with pipeline: - Join()(src_a, src_b, label="joined") - assert pipeline.function_pods == {} - - def test_operator_pods_empty_when_no_operator_nodes(self): - src = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - pipeline = Pipeline(name="test") - with pipeline: - pod(src, label="doubled") - assert pipeline.operator_pods == {} -``` - -- [ ] **Step 2: Run the new tests — confirm they all fail with `AttributeError`** - -```bash -uv run pytest tests/test_pipeline/test_pipeline.py::TestTypePodAccess -v -``` - -Expected: 6 FAILED — each with: -``` -AttributeError: 'Pipeline' object has no attribute 'function_pods' -``` -(or `source_pods` / `operator_pods` depending on which test runs first) - -- [ ] **Step 3: Commit the failing tests** - -```bash -git add tests/test_pipeline/test_pipeline.py -git commit -m "test(pipeline): add failing tests for source_pods, function_pods, operator_pods" -``` - ---- - -## Task 2: Implement properties on `AbstractPipelineBase` - -**Files:** -- Modify: `src/orcapod/pipeline/base.py:118-121` (after the `.nodes` property) - -- [ ] **Step 1: Add the three properties immediately after the `nodes` property in `base.py`** - -Locate this block (lines ~118–121): - -```python - @property - def nodes(self) -> dict[str, Any]: - """Copy of the compiled nodes dict (label → node).""" - return self._nodes.copy() -``` - -Insert the following three properties directly after it (before the `dag` property): - -```python - @property - def source_pods(self) -> dict[str, Any]: - """Copy of compiled nodes that are source nodes (label → node).""" - return {k: v for k, v in self._nodes.items() if v.node_type == "source"} - - @property - def function_pods(self) -> dict[str, Any]: - """Copy of compiled nodes that are function-pod nodes (label → node).""" - return {k: v for k, v in self._nodes.items() if v.node_type == "function"} - - @property - def operator_pods(self) -> dict[str, Any]: - """Copy of compiled nodes that are operator-pod nodes (label → node).""" - return {k: v for k, v in self._nodes.items() if v.node_type == "operator"} -``` - -- [ ] **Step 2: Run the new tests — confirm they all pass** - -```bash -uv run pytest tests/test_pipeline/test_pipeline.py::TestTypePodAccess -v -``` - -Expected: 6 PASSED - -- [ ] **Step 3: Run the full pipeline test suite — confirm no regressions** - -```bash -uv run pytest tests/test_pipeline/ -v -``` - -Expected: all tests PASSED (no failures introduced) - -- [ ] **Step 4: Commit the implementation** - -```bash -git add src/orcapod/pipeline/base.py -git commit -m "feat(pipeline): add source_pods, function_pods, operator_pods properties to AbstractPipelineBase" -``` - ---- - -## Task 3: Update `PipelineProtocol` - -**Files:** -- Modify: `src/orcapod/protocols/pipeline_protocols.py:38-45` (after the `.nodes` property stub) - -- [ ] **Step 1: Add three property stubs to `PipelineProtocol` after the `nodes` property** - -Locate this block in `PipelineProtocol`: - -```python - @property - def nodes(self) -> dict[str, NodeT]: - """Copy of the compiled label -> node mapping.""" - ... -``` - -Insert the following three stubs directly after it (before the `dag` property): - -```python - @property - def source_pods(self) -> dict[str, NodeT]: - """Copy of compiled nodes that are source nodes (label → node).""" - ... - - @property - def function_pods(self) -> dict[str, NodeT]: - """Copy of compiled nodes that are function-pod nodes (label → node).""" - ... - - @property - def operator_pods(self) -> dict[str, NodeT]: - """Copy of compiled nodes that are operator-pod nodes (label → node).""" - ... -``` - -- [ ] **Step 2: Run the full test suite — confirm everything still passes** - -```bash -uv run pytest tests/ -v -``` - -Expected: all tests PASSED - -- [ ] **Step 3: Commit the protocol update** - -```bash -git add src/orcapod/protocols/pipeline_protocols.py -git commit -m "feat(protocols): add source_pods, function_pods, operator_pods to PipelineProtocol" -``` diff --git a/superpowers/plans/2026-06-13-datetime-arrow-timestamp.md b/superpowers/plans/2026-06-13-datetime-arrow-timestamp.md deleted file mode 100644 index 1f5c6b313..000000000 --- a/superpowers/plans/2026-06-13-datetime-arrow-timestamp.md +++ /dev/null @@ -1,438 +0,0 @@ -# datetime ↔ Arrow timestamp support Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add native `datetime.datetime ↔ pa.timestamp("us", tz="UTC")` support to `UniversalTypeConverter` and revert the ISO string workaround in `ResultCache`. - -**Architecture:** Four surgical additions to `universal_converter.py` (type map entry, reverse type branch, Python→Arrow value validator, Arrow→Python explicit passthrough) plus a one-line revert in `result_cache.py`. All changes are tested with TDD before implementation. - -**Tech Stack:** Python 3.11+, PyArrow ≥ 20, pytest, `uv run pytest` - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/semantic_types/universal_converter.py` | Add `datetime` import; 4 targeted additions | -| `src/orcapod/core/result_cache.py` | Revert ISO string workaround to native `pa.timestamp` | -| `tests/test_semantic_types/test_universal_converter.py` | Add 7 new tests | - ---- - -## Task 1: Type mapping — Python → Arrow and Arrow → Python - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py:12-32` (imports), `:61-92` (`_get_python_to_arrow_map`), `:637-650` (`_convert_arrow_to_python` else-fallback) -- Test: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1.1: Write three failing type-mapping tests** - -Open `tests/test_semantic_types/test_universal_converter.py`. Add the following import at the top of the file alongside the existing imports: - -```python -from datetime import datetime, timezone -``` - -Then add these three test functions after `test_python_type_to_arrow_type_basic`: - -```python -def test_python_type_to_arrow_type_datetime(): - assert universal_converter.python_type_to_arrow_type(datetime) == pa.timestamp( - "us", tz="UTC" - ) - - -def test_arrow_type_to_python_type_timestamp_with_tz(): - assert ( - universal_converter.arrow_type_to_python_type(pa.timestamp("us", tz="UTC")) - is datetime - ) - - -def test_arrow_type_to_python_type_timestamp_no_tz(): - assert universal_converter.arrow_type_to_python_type(pa.timestamp("us")) is datetime -``` - -- [ ] **Step 1.2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_python_type_to_arrow_type_datetime tests/test_semantic_types/test_universal_converter.py::test_arrow_type_to_python_type_timestamp_with_tz tests/test_semantic_types/test_universal_converter.py::test_arrow_type_to_python_type_timestamp_no_tz -v -``` - -Expected: all three FAIL — `ValueError: Unsupported Python type` for the first, and the last two return `Any` instead of `datetime`. - -- [ ] **Step 1.3: Add the `datetime` import to `universal_converter.py`** - -In `src/orcapod/semantic_types/universal_converter.py`, the current imports block starts at line 12 with `from __future__ import annotations`. Add the stdlib import on line 14 (after the `from __future__` line and the blank line): - -Find this block: -```python -from __future__ import annotations - -import hashlib -import logging -import types -import typing -``` - -Replace with: -```python -from __future__ import annotations - -import hashlib -import logging -import types -import typing -from datetime import datetime, timezone -``` - -- [ ] **Step 1.4: Add `datetime` to `_get_python_to_arrow_map()`** - -In `_get_python_to_arrow_map()`, find the date/time string entries (around line 88): - -```python - # Date/time types - "date": pa.date32(), - "datetime": pa.timestamp("us"), - "timestamp": pa.timestamp("us"), -``` - -Replace with: - -```python - # Date/time types - "date": pa.date32(), - "datetime": pa.timestamp("us"), - "timestamp": pa.timestamp("us"), - datetime: pa.timestamp("us", tz="UTC"), -``` - -- [ ] **Step 1.5: Add `is_timestamp` branch to `_convert_arrow_to_python()`** - -Find the `else` fallback at line 637: - -```python - else: - # Default case for unsupported types. - # NOTE: this silent fallback to Any can cause cryptic errors - # downstream when code tries to convert Any back to Arrow - # (e.g. "Unsupported Python type: typing.Any"). If you hit that, - # the root cause is likely an unmapped Arrow type here. - # (pa.null() is intentionally excluded — it is handled above.) - logger.warning( -``` - -Insert the new branch immediately before it: - -```python - elif pa.types.is_timestamp(arrow_type): - return datetime - - else: - # Default case for unsupported types. -``` - -- [ ] **Step 1.6: Run the three tests to confirm they pass** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_python_type_to_arrow_type_datetime tests/test_semantic_types/test_universal_converter.py::test_arrow_type_to_python_type_timestamp_with_tz tests/test_semantic_types/test_universal_converter.py::test_arrow_type_to_python_type_timestamp_no_tz -v -``` - -Expected: all three PASS. - -- [ ] **Step 1.7: Run the full converter test suite to check for regressions** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -``` - -Expected: all existing tests still pass. - -- [ ] **Step 1.8: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "feat(semantic-types): map datetime ↔ pa.timestamp(us, UTC) in type converter" -``` - ---- - -## Task 2: Value converters — naive rejection and explicit Arrow → Python passthrough - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py:748-753` (`_create_python_to_arrow_converter`), `:929-931` (`_create_arrow_to_python_converter`) -- Test: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 2.1: Write two failing value-converter tests** - -Add these two tests after `test_arrow_type_to_python_type_timestamp_no_tz`: - -```python -def test_datetime_converter_rejects_naive(): - to_arrow, _ = universal_converter.get_conversion_functions(datetime) - naive = datetime(2024, 1, 15, 12, 30, 45, 123456) # no tzinfo - with pytest.raises(ValueError, match="Naive datetime"): - to_arrow(naive) - - -def test_datetime_converter_accepts_aware(): - to_arrow, _ = universal_converter.get_conversion_functions(datetime) - aware = datetime(2024, 1, 15, 12, 30, 45, 123456, tzinfo=timezone.utc) - result = to_arrow(aware) - assert result == aware -``` - -- [ ] **Step 2.2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_datetime_converter_rejects_naive tests/test_semantic_types/test_universal_converter.py::test_datetime_converter_accepts_aware -v -``` - -Expected: `test_datetime_converter_rejects_naive` FAILS (no `ValueError` raised — naive datetime silently passes through), `test_datetime_converter_accepts_aware` MAY pass or fail. - -- [ ] **Step 2.3: Add the datetime validator in `_create_python_to_arrow_converter()`** - -In `_create_python_to_arrow_converter()`, find the block starting at line 746: - -```python - # Create conversion function based on type - - origin = get_origin(python_type) - args = get_args(python_type) - - if python_type in {int, float, str, bool, bytes} or origin is None: - # Basic types - no conversion needed - return lambda value: value -``` - -Replace with: - -```python - # Create conversion function based on type - - # datetime must be intercepted before the `origin is None` catch-all - # below, which would silently pass through naive datetimes. - if python_type is datetime: - - def _convert_datetime(dt: datetime) -> datetime: - if dt.tzinfo is None: - raise ValueError( - f"Naive datetime (no timezone info) is not supported. " - f"Use a timezone-aware datetime, " - f"e.g. datetime.now(timezone.utc). Got: {dt!r}" - ) - return dt - - return _convert_datetime - - origin = get_origin(python_type) - args = get_args(python_type) - - if python_type in {int, float, str, bool, bytes} or origin is None: - # Basic types - no conversion needed - return lambda value: value -``` - -- [ ] **Step 2.4: Add explicit timestamp passthrough in `_create_arrow_to_python_converter()`** - -Find the `else` passthrough at line 929: - -```python - else: - # Default passthrough - return lambda value: value -``` - -Insert the new branch immediately before it: - -```python - elif pa.types.is_timestamp(arrow_type): - # PyArrow's to_pylist() already calls .as_py() on each scalar, - # returning a timezone-aware datetime for UTC columns. - return lambda value: value - - else: - # Default passthrough - return lambda value: value -``` - -- [ ] **Step 2.5: Run the two value-converter tests to confirm they pass** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_datetime_converter_rejects_naive tests/test_semantic_types/test_universal_converter.py::test_datetime_converter_accepts_aware -v -``` - -Expected: both PASS. - -- [ ] **Step 2.6: Run the full converter test suite to check for regressions** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 2.7: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "feat(semantic-types): add datetime value converter with naive-datetime guard" -``` - ---- - -## Task 3: Round-trip tests — full table conversion - -**Files:** -- Test: `tests/test_semantic_types/test_universal_converter.py` - -These tests exercise `python_dicts_to_arrow_table` and `arrow_table_to_python_dicts` on the -global converter retrieved via `get_default_context().type_converter`. - -- [ ] **Step 3.1: Write two round-trip tests** - -`get_default_context` is already imported at the top of the test file — no new import needed. - -Add these two tests after `test_datetime_converter_accepts_aware`: - -```python -def test_datetime_round_trip(): - converter = get_default_context().type_converter - ts = datetime(2024, 3, 15, 10, 30, 45, 123456, tzinfo=timezone.utc) - rows_in = [{"event": "launch", "ts": ts}] - - table = converter.python_dicts_to_arrow_table(rows_in) - - # Arrow schema must use timestamp(us, UTC) - assert table.schema.field("ts").type == pa.timestamp("us", tz="UTC") - - rows_out = converter.arrow_table_to_python_dicts(table) - assert len(rows_out) == 1 - assert rows_out[0]["event"] == "launch" - assert rows_out[0]["ts"] == ts - - -def test_optional_datetime_round_trip(): - converter = get_default_context().type_converter - ts = datetime(2024, 6, 1, 0, 0, 0, tzinfo=timezone.utc) - rows_in = [ - {"label": "a", "ts": ts}, - {"label": "b", "ts": None}, - ] - python_schema = {"label": str, "ts": datetime | None} - - table = converter.python_dicts_to_arrow_table(rows_in, python_schema=python_schema) - - assert table.schema.field("ts").type == pa.timestamp("us", tz="UTC") - assert table.schema.field("ts").nullable is True - - rows_out = converter.arrow_table_to_python_dicts(table) - assert rows_out[0]["ts"] == ts - assert rows_out[1]["ts"] is None -``` - -- [ ] **Step 3.2: Run the tests — they should pass immediately** - -Tasks 1 and 2 already provide all the implementation needed for these tests. Run them to -confirm end-to-end correctness: - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_datetime_round_trip tests/test_semantic_types/test_universal_converter.py::test_optional_datetime_round_trip -v -``` - -Expected: both PASS. - -If either test fails, debug using the failing assertion message — the type mapping or value -converter from Tasks 1/2 may need a correction. - -- [ ] **Step 3.4: Run the full converter test suite** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 3.5: Commit** - -```bash -git add tests/test_semantic_types/test_universal_converter.py -git commit -m "test(semantic-types): add datetime ↔ Arrow timestamp round-trip tests" -``` - ---- - -## Task 4: Revert `ResultCache` ISO string workaround - -**Files:** -- Modify: `src/orcapod/core/result_cache.py:194-203` - -- [ ] **Step 4.1: Run the existing result_cache tests to establish baseline** - -```bash -uv run pytest tests/test_core/test_result_cache.py -v -``` - -Note the number of passing tests as your baseline. - -- [ ] **Step 4.2: Revert the ISO string workaround** - -In `src/orcapod/core/result_cache.py`, find lines 194–203: - -```python - # Append timestamp as ISO 8601 string. - # TODO: switch to pa.timestamp("us", tz="UTC") once the universal - # converter supports native Arrow timestamp ↔ Python datetime round-trip. - # ISO 8601 strings sort lexicographically in time order, so the - # conflict-resolution sort in lookup() still works correctly. - timestamp = datetime.now(timezone.utc).isoformat() - data_table = data_table.append_column( - constants.POD_TIMESTAMP, - pa.array([timestamp], type=pa.large_string()), - ) -``` - -Replace with: - -```python - data_table = data_table.append_column( - constants.POD_TIMESTAMP, - pa.array([datetime.now(timezone.utc)], type=pa.timestamp("us", tz="UTC")), - ) -``` - -- [ ] **Step 4.3: Run the result_cache tests to confirm no regressions** - -```bash -uv run pytest tests/test_core/test_result_cache.py -v -``` - -Expected: same number of passing tests as the baseline from Step 4.1. - -- [ ] **Step 4.4: Run the full test suite** - -```bash -uv run pytest -v -``` - -Expected: all tests pass. No new failures introduced by the timestamp type change. - -- [ ] **Step 4.5: Commit** - -```bash -git add src/orcapod/core/result_cache.py -git commit -m "fix(core): revert ResultCache timestamp to native pa.timestamp(us, UTC) — ENG-387" -``` - ---- - -## Final check - -- [ ] **Run the complete test suite one last time** - -```bash -uv run pytest -v -``` - -Expected: all tests pass, zero failures. diff --git a/superpowers/plans/2026-06-13-plt-1652-extension-type-converter-protocol.md b/superpowers/plans/2026-06-13-plt-1652-extension-type-converter-protocol.md deleted file mode 100644 index 6dfb405ee..000000000 --- a/superpowers/plans/2026-06-13-plt-1652-extension-type-converter-protocol.md +++ /dev/null @@ -1,393 +0,0 @@ -# ExtensionTypeConverter Protocol Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Create `src/orcapod/extension_types/protocols.py` containing the `ExtensionTypeConverter` Protocol, establishing the new `extension_types/` subpackage. - -**Architecture:** New code only — the existing `SemanticStructConverterProtocol` and all old system files are untouched (parallel-build strategy). The `extension_types/` subpackage is created fresh. Only two files are written: the package `__init__.py` (empty placeholder) and `protocols.py` containing `ExtensionTypeConverter`. - -**Tech Stack:** Python `typing.Protocol` (Python ≥ 3.11), PyArrow (`pa.DataType`), `uv run pytest` - ---- - -## File map - -| Action | Path | Responsibility | -|---|---|---| -| Create | `src/orcapod/extension_types/__init__.py` | Empty package marker (populated by PLT-1656) | -| Create | `src/orcapod/extension_types/protocols.py` | `ExtensionTypeConverter` protocol | -| Create | `tests/test_extension_types/__init__.py` | Empty test package marker | -| Create | `tests/test_extension_types/test_protocols.py` | All protocol conformance tests | - ---- - -## Task 0: Checkout the feature branch - -- [ ] **Step 0.1: Create and check out the branch** - -```bash -git checkout -b eywalker/plt-1652-define-extensiontypeconverter-protocol-with-extension-name -``` - -Expected output: `Switched to a new branch 'eywalker/plt-1652-...'` - -- [ ] **Step 0.2: Verify** - -```bash -git branch --show-current -``` - -Expected: `eywalker/plt-1652-define-extensiontypeconverter-protocol-with-extension-name` - ---- - -## Task 1: Scaffold the package structure - -**Files:** -- Create: `src/orcapod/extension_types/__init__.py` -- Create: `tests/test_extension_types/__init__.py` - -- [ ] **Step 1.1: Create the `extension_types` source package** - -```bash -mkdir -p src/orcapod/extension_types -touch src/orcapod/extension_types/__init__.py -``` - -The `__init__.py` stays empty for now. PLT-1656 will populate it with import-time converter registration. - -- [ ] **Step 1.2: Create the test package** - -```bash -mkdir -p tests/test_extension_types -touch tests/test_extension_types/__init__.py -``` - -- [ ] **Step 1.3: Verify the directories exist** - -```bash -ls src/orcapod/extension_types/ && ls tests/test_extension_types/ -``` - -Expected: -``` -__init__.py -__init__.py -``` - ---- - -## Task 2: Write failing tests - -**Files:** -- Create: `tests/test_extension_types/test_protocols.py` - -- [ ] **Step 2.1: Write the test file** - -Create `tests/test_extension_types/test_protocols.py` with this exact content: - -```python -"""Tests for ExtensionTypeConverter protocol.""" - -from __future__ import annotations - -import pyarrow as pa - -from orcapod.extension_types.protocols import ExtensionTypeConverter - - -class _StubConverter: - """Minimal conforming implementation of ExtensionTypeConverter for use in tests.""" - - @property - def extension_name(self) -> str: - return "test.module.MyType" - - @property - def extension_metadata(self) -> bytes | None: - return b"test.category" - - @property - def storage_type(self) -> pa.DataType: - return pa.large_string() - - @property - def python_type(self) -> type: - return str - - def python_to_storage(self, value): - return str(value) - - def storage_to_python(self, storage_value): - return storage_value - - -def test_protocol_is_importable(): - """ExtensionTypeConverter can be imported from extension_types.protocols.""" - assert ExtensionTypeConverter is not None - - -def test_protocol_defines_required_members(): - """Protocol defines all six required members.""" - required = { - "extension_name", - "extension_metadata", - "storage_type", - "python_type", - "python_to_storage", - "storage_to_python", - } - for member in required: - assert hasattr(ExtensionTypeConverter, member), f"Protocol missing member: {member}" - - -def test_conforming_class_satisfies_protocol(): - """A class implementing all required members works correctly via the protocol interface.""" - converter: ExtensionTypeConverter = _StubConverter() - assert converter.extension_name == "test.module.MyType" - assert converter.extension_metadata == b"test.category" - assert converter.storage_type == pa.large_string() - assert converter.python_type is str - assert converter.python_to_storage(42) == "42" - assert converter.storage_to_python("hello") == "hello" - - -def test_extension_metadata_can_be_none(): - """extension_metadata is allowed to be None — it is bytes | None.""" - - class NullMetadataConverter: - @property - def extension_name(self) -> str: - return "test.NullMeta" - - @property - def extension_metadata(self) -> bytes | None: - return None - - @property - def storage_type(self) -> pa.DataType: - return pa.binary(16) - - @property - def python_type(self) -> type: - return bytes - - def python_to_storage(self, value): - return value - - def storage_to_python(self, storage_value): - return storage_value - - converter: ExtensionTypeConverter = NullMetadataConverter() - assert converter.extension_metadata is None - - -def test_storage_type_not_constrained_to_struct(): - """storage_type accepts any pa.DataType — primitive types are valid, not only struct.""" - - class BinaryConverter: - @property - def extension_name(self) -> str: - return "uuid.UUID" - - @property - def extension_metadata(self) -> bytes | None: - return b"orcapod.builtin" - - @property - def storage_type(self) -> pa.DataType: - return pa.binary(16) - - @property - def python_type(self) -> type: - return bytes - - def python_to_storage(self, value): - return value - - def storage_to_python(self, storage_value): - return bytes(storage_value) - - converter: ExtensionTypeConverter = BinaryConverter() - assert converter.storage_type == pa.binary(16) - assert not pa.types.is_struct(converter.storage_type) - - -def test_protocol_does_not_include_old_members(): - """The new protocol must not define hashing or struct-dispatch members from the old protocol.""" - excluded = { - "hash_struct_dict", - "hasher_id", - "can_handle_python_type", - "can_handle_struct_type", - "arrow_struct_type", - } - for member in excluded: - assert not hasattr(ExtensionTypeConverter, member), ( - f"ExtensionTypeConverter must not define '{member}' — " - "hashing and struct-shape dispatch are not part of the new protocol" - ) -``` - -- [ ] **Step 2.2: Run the tests to confirm they fail with ImportError** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: all tests **FAIL** with: -``` -ModuleNotFoundError: No module named 'orcapod.extension_types' -``` -(or `ImportError: cannot import name 'ExtensionTypeConverter'` once the package exists but the file does not) - ---- - -## Task 3: Implement ExtensionTypeConverter - -**Files:** -- Create: `src/orcapod/extension_types/protocols.py` - -- [ ] **Step 3.1: Write the protocol** - -Create `src/orcapod/extension_types/protocols.py` with this exact content: - -```python -"""Protocol definitions for the Arrow/Polars extension type system. - -This module defines ``ExtensionTypeConverter`` — the contract for all -converters that map between Python objects and their Arrow extension type -storage representation. - -Note: - This module is part of the parallel-build phase. The old - ``SemanticStructConverterProtocol`` in ``protocols/semantic_types_protocols.py`` - is untouched; it is removed in PLT-1660. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol - -if TYPE_CHECKING: - import pyarrow as pa - - -class ExtensionTypeConverter(Protocol): - """Protocol for Arrow/Polars extension-type-backed converters. - - Declares the full contract for a converter that maps between Python - objects and their Arrow extension type storage representation. This - protocol is Arrow I/O only — hashing is not a converter responsibility. - - Attributes: - extension_name: Fully-qualified Python class name used as the - ``ARROW:extension:name`` metadata value (e.g. ``"pathlib.Path"``). - Must be unique across all registered converters. By convention - equals the FQCN, but any unique string is valid. - extension_metadata: Category tag encoded as ``ARROW:extension:metadata`` - (e.g. ``b"orcapod.dataclass"``). Used by the registry to locate - the right category handler at read time. May be ``None``. - storage_type: The underlying Arrow ``pa.DataType`` used for physical - storage (e.g. ``pa.large_string()``, ``pa.binary(16)``, - ``pa.struct(...)``). Not used as an identity signal — identity - is determined solely by ``extension_name``. - python_type: The Python class this converter handles. - """ - - @property - def extension_name(self) -> str: - """Fully-qualified Python class name; stored as ``ARROW:extension:name``.""" - ... - - @property - def extension_metadata(self) -> "bytes | None": - """Category tag; stored as ``ARROW:extension:metadata``. May be ``None``.""" - ... - - @property - def storage_type(self) -> "pa.DataType": - """Underlying Arrow storage type. Any ``pa.DataType`` is valid.""" - ... - - @property - def python_type(self) -> type: - """The Python class this converter handles.""" - ... - - def python_to_storage(self, value: Any) -> Any: - """Convert a Python value to its Arrow storage representation. - - Args: - value: A Python object of type ``python_type``. - - Returns: - A value suitable for use as an Arrow scalar or array element - of type ``storage_type``. - """ - ... - - def storage_to_python(self, storage_value: Any) -> Any: - """Convert an Arrow storage value back to a Python object. - - Args: - storage_value: A scalar or array element of type ``storage_type``. - - Returns: - A Python object of type ``python_type``. - """ - ... -``` - -- [ ] **Step 3.2: Run the full test suite to confirm tests pass and nothing regresses** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: all 6 tests **PASS**: -``` -tests/test_extension_types/test_protocols.py::test_protocol_is_importable PASSED -tests/test_extension_types/test_protocols.py::test_protocol_defines_required_members PASSED -tests/test_extension_types/test_protocols.py::test_conforming_class_satisfies_protocol PASSED -tests/test_extension_types/test_protocols.py::test_extension_metadata_can_be_none PASSED -tests/test_extension_types/test_protocols.py::test_storage_type_not_constrained_to_struct PASSED -tests/test_extension_types/test_protocols.py::test_protocol_does_not_include_old_members PASSED -``` - -- [ ] **Step 3.3: Run the full test suite to verify nothing is broken** - -```bash -uv run pytest --ignore=tests/test_databases -x -q -``` - -Expected: all existing tests still pass. (Databases tests are excluded as they require live connections.) - ---- - -## Task 4: Commit - -- [ ] **Step 4.1: Stage and commit** - -```bash -git add src/orcapod/extension_types/__init__.py \ - src/orcapod/extension_types/protocols.py \ - tests/test_extension_types/__init__.py \ - tests/test_extension_types/test_protocols.py \ - superpowers/specs/2026-06-13-plt-1652-extension-type-converter-protocol-design.md \ - docs/metamorphic/plans/2026-06-13-plt-1652-extension-type-converter-protocol.md - -git commit -m "feat(extension_types): add ExtensionTypeConverter protocol and extension_types subpackage - -Establishes the extension_types/ subpackage and defines ExtensionTypeConverter — -the new Arrow I/O-only protocol for extension-type-backed converters (PLT-1652). - -Old SemanticStructConverterProtocol is not touched (parallel-build strategy). -Removal is deferred to PLT-1660." -``` - -- [ ] **Step 4.2: Push the branch** - -```bash -git push -u origin eywalker/plt-1652-define-extensiontypeconverter-protocol-with-extension-name -``` diff --git a/superpowers/plans/2026-06-14-extension-type-registry.md b/superpowers/plans/2026-06-14-extension-type-registry.md deleted file mode 100644 index 14a119f53..000000000 --- a/superpowers/plans/2026-06-14-extension-type-registry.md +++ /dev/null @@ -1,925 +0,0 @@ -# ExtensionTypeRegistry Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement `ExtensionTypeRegistry` in `src/orcapod/extension_types/registry.py`, wiring up both PyArrow and Polars global extension type registries on each `register()` call. - -**Architecture:** A plain Python class with two internal dicts (`_by_name`, `_by_python_type`) for converter lookup, plus two module-level shadow dicts (`_ARROW_REGISTRY`, `_POLARS_REGISTRY`) that track what has been registered in the process-global PA/Polars registries. `register()` validates against both the instance dict (duplicate check) and the shadow dicts (equivalence/external-conflict check), then dynamically creates and registers `pa.ExtensionType` and `pl.BaseExtension` subclasses via `type()`. - -**Tech Stack:** Python 3.12, PyArrow ≥ 20, Polars ≥ 1.36, pytest, uv - -**Spec:** `superpowers/specs/2026-06-14-extension-type-registry-design.md` - ---- - -## File map - -| File | Action | -|---|---| -| `pyproject.toml` | Modify — restore range constraint `polars>=1.36.0` | -| `src/orcapod/extension_types/registry.py` | **Create** — `ExtensionTypeRegistry`, shadow dicts, helpers | -| `src/orcapod/extension_types/__init__.py` | Modify — export class, create module-level instance | -| `tests/test_extension_types/test_registry.py` | **Create** — full test suite | - ---- - -## Task 1: Fix `pyproject.toml` — restore Polars range constraint - -The Polars dependency was accidentally pinned to `==1.41.2` during exploration. Restore it to a range constraint. - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Update the constraint** - -Open `pyproject.toml`. Find the line: -```toml -"polars==1.41.2", -``` -Replace with: -```toml -"polars>=1.36.0", -``` - -- [ ] **Step 2: Sync and verify** - -```bash -uv sync -uv run python -c "import polars as pl; print(pl.__version__); from polars import BaseExtension; print('BaseExtension OK')" -``` - -Expected output: -``` -1.41.2 -BaseExtension OK -``` - -- [ ] **Step 3: Commit** - -```bash -git add pyproject.toml uv.lock -git commit -m "chore(deps): restore polars>=1.36.0 range constraint (PLT-1653)" -``` - ---- - -## Task 2: Create `test_registry.py` and `registry.py` — pure-Python registry - -Write all tests that exercise the Python-only layer (dict storage, lookups, duplicate checking). No PA/Polars wiring yet — `register()` just populates the internal dicts. - -**Files:** -- Create: `tests/test_extension_types/test_registry.py` -- Create: `src/orcapod/extension_types/registry.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_extension_types/test_registry.py`: - -```python -"""Tests for ExtensionTypeRegistry.""" - -from __future__ import annotations - -import uuid - -import pyarrow as pa -import pytest - -from orcapod.extension_types.protocols import ExtensionTypeConverter -from orcapod.extension_types.registry import ExtensionTypeRegistry - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _unique_name() -> str: - """Unique extension name to avoid cross-test global-registry collisions.""" - return f"test.registry.{uuid.uuid4().hex[:8]}" - - -def _make_stub( - name: str | None = None, - storage: pa.DataType | None = None, - metadata: bytes | None = b"test.category", - py_type: type = str, -) -> ExtensionTypeConverter: - """Factory for minimal ExtensionTypeConverter conforming stubs.""" - _name = name or _unique_name() - _storage = storage if storage is not None else pa.large_utf8() - _metadata = metadata - _py_type = py_type - - class _Stub: - @property - def extension_name(self) -> str: - return _name - - @property - def extension_metadata(self) -> bytes | None: - return _metadata - - @property - def storage_type(self) -> pa.DataType: - return _storage - - @property - def python_type(self) -> type: - return _py_type - - def python_to_storage(self, value): - return str(value) - - def storage_to_python(self, storage_value): - return storage_value - - return _Stub() - - -# --------------------------------------------------------------------------- -# Pure-Python registry tests (no PA/Polars global state required) -# --------------------------------------------------------------------------- - -def test_register_stores_converter(): - registry = ExtensionTypeRegistry() - conv = _make_stub() - registry.register(conv) - assert registry.get_converter_for_name(conv.extension_name) is conv - - -def test_register_duplicate_raises(): - registry = ExtensionTypeRegistry() - name = _unique_name() - registry.register(_make_stub(name=name)) - with pytest.raises(ValueError, match=name): - registry.register(_make_stub(name=name)) - - -def test_get_converter_for_name_miss(): - registry = ExtensionTypeRegistry() - assert registry.get_converter_for_name("does.not.exist") is None - - -def test_get_converter_for_python_type_exact(): - registry = ExtensionTypeRegistry() - conv = _make_stub(py_type=bytes) - registry.register(conv) - assert registry.get_converter_for_python_type(bytes) is conv - - -def test_get_converter_for_python_type_subclass(): - class _Base: - pass - - class _Child(_Base): - pass - - registry = ExtensionTypeRegistry() - conv = _make_stub(py_type=_Base) - registry.register(conv) - assert registry.get_converter_for_python_type(_Child) is conv - - -def test_get_converter_for_python_type_miss(): - registry = ExtensionTypeRegistry() - assert registry.get_converter_for_python_type(int) is None - - -def test_has_extension_name(): - registry = ExtensionTypeRegistry() - conv = _make_stub() - assert not registry.has_extension_name(conv.extension_name) - registry.register(conv) - assert registry.has_extension_name(conv.extension_name) - - -def test_has_python_type(): - registry = ExtensionTypeRegistry() - conv = _make_stub(py_type=float) - assert not registry.has_python_type(float) - registry.register(conv) - assert registry.has_python_type(float) - - -def test_list_extension_names(): - registry = ExtensionTypeRegistry() - a = _make_stub() - b = _make_stub() - registry.register(a) - registry.register(b) - assert registry.list_extension_names() == [a.extension_name, b.extension_name] - - -def test_list_python_types(): - registry = ExtensionTypeRegistry() - a = _make_stub(py_type=bytes) - b = _make_stub(py_type=float) - registry.register(a) - registry.register(b) - assert registry.list_python_types() == [bytes, float] -``` - -- [ ] **Step 2: Run to confirm ImportError (registry module does not exist yet)** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v 2>&1 | head -20 -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.extension_types.registry'` - -- [ ] **Step 3: Create `src/orcapod/extension_types/registry.py`** - -```python -"""Registry for ExtensionTypeConverter instances. - -Registering a converter automatically registers the corresponding -extension type in both PyArrow's and Polars' global registries. -""" - -from __future__ import annotations - -import re - -import pyarrow as pa -import polars as pl - -from orcapod.extension_types.protocols import ExtensionTypeConverter - -# --------------------------------------------------------------------------- -# Shadow dicts — track what *we* have registered in the global registries. -# These are module-level singletons shared across all ExtensionTypeRegistry -# instances. We use our own dicts rather than querying library internals -# because neither PyArrow nor Polars exposes a stable public API for looking -# up a previously registered extension type by name. -# -# Limitation: types registered externally (directly via -# pa.register_extension_type / pl.register_extension_type, bypassing this -# module) will not appear here. A subsequent register() call for the same -# name will detect the conflict via the library-level error and raise, -# because without knowing what was registered externally we cannot guarantee -# the same extension name maps to the same Python class and underlying -# storage type — silently proceeding risks data corruption or misrouted -# conversions at read time. -# --------------------------------------------------------------------------- - -_ARROW_REGISTRY: dict[str, tuple[pa.DataType, bytes]] = {} -# extension_name -> (storage_type, metadata_bytes) - -_POLARS_REGISTRY: dict[str, tuple[pl.DataType, str | None]] = {} -# extension_name -> (pl_storage_dtype, metadata_str) - - -def _sanitize(name: str) -> str: - return re.sub(r"[^A-Za-z0-9]", "_", name) - - -def _register_arrow_ext_type(converter: ExtensionTypeConverter) -> None: - """Register a ``pa.ExtensionType`` subclass for *converter* in PyArrow's global registry.""" - name = converter.extension_name - metadata = converter.extension_metadata or b"" - storage = converter.storage_type - - if name in _ARROW_REGISTRY: - existing_storage, existing_metadata = _ARROW_REGISTRY[name] - if existing_storage == storage and existing_metadata == metadata: - return # idempotent — safe for module reload and test-suite reuse - raise ValueError( - f"Extension type '{name}' is already registered in the PyArrow global registry " - f"with different parameters.\n" - f" Registered: storage_type={existing_storage!r}, metadata={existing_metadata!r}\n" - f" Attempted: storage_type={storage!r}, metadata={metadata!r}" - ) - - _name, _storage, _metadata = name, storage, metadata - ArrowExtType = type( - f"_ArrowExt_{_sanitize(name)}", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _storage, _name), - "__arrow_ext_serialize__": lambda self: _metadata, - "__arrow_ext_deserialize__": classmethod(lambda cls, st, se: cls()), - }, - ) - - try: - pa.register_extension_type(ArrowExtType()) - except pa.lib.ArrowKeyError: - raise ValueError( - f"Extension type '{name}' is already registered in the PyArrow global registry " - f"by an external source. Cannot verify equivalence; orcapod requires exclusive " - f"ownership of extension type registrations to prevent data corruption or " - f"misrouted conversions. See PLT-1665 for future interop support." - ) from None - - _ARROW_REGISTRY[name] = (storage, metadata) - - -def _register_polars_ext_type(converter: ExtensionTypeConverter) -> None: - """Register a ``pl.BaseExtension`` subclass for *converter* in Polars' global registry.""" - name = converter.extension_name - metadata = converter.extension_metadata - metadata_str = metadata.decode("utf-8") if metadata else None - pl_storage = pl.from_arrow(pa.array([], type=converter.storage_type)).dtype - - if name in _POLARS_REGISTRY: - existing_storage, existing_meta = _POLARS_REGISTRY[name] - if existing_storage == pl_storage and existing_meta == metadata_str: - return # idempotent - raise ValueError( - f"Extension type '{name}' is already registered in the Polars global registry " - f"with different parameters.\n" - f" Registered: storage_dtype={existing_storage!r}, metadata={existing_meta!r}\n" - f" Attempted: storage_dtype={pl_storage!r}, metadata={metadata_str!r}" - ) - - _name, _pl_storage, _meta_str = name, pl_storage, metadata_str - PolarsExtType = type( - f"_PolarsExt_{_sanitize(name)}", - (pl.BaseExtension,), - { - "__init__": lambda self: pl.BaseExtension.__init__(self, _name, _pl_storage, _meta_str), - "ext_from_params": classmethod(lambda cls, n, s, m: cls()), - }, - ) - - try: - pl.register_extension_type(name, PolarsExtType) - except ValueError as exc: - raise ValueError( - f"Extension type '{name}' is already registered in the Polars global registry " - f"by an external source. Cannot verify equivalence; orcapod requires exclusive " - f"ownership of extension type registrations to prevent data corruption or " - f"misrouted conversions. See PLT-1665 for future interop support." - ) from exc - - _POLARS_REGISTRY[name] = (pl_storage, metadata_str) - - -class ExtensionTypeRegistry: - """Registry for ``ExtensionTypeConverter`` instances. - - Registering a converter automatically registers the corresponding - extension type in both PyArrow's and Polars' global registries. - - The primary lookup key is ``extension_name``; a secondary lookup by - ``python_type`` is provided for the write path. - - Example: - >>> registry = ExtensionTypeRegistry() - >>> registry.register(my_converter) - >>> conv = registry.get_converter_for_name("my.Type") - """ - - def __init__(self) -> None: - self._by_name: dict[str, ExtensionTypeConverter] = {} - self._by_python_type: dict[type, ExtensionTypeConverter] = {} - - def register(self, converter: ExtensionTypeConverter) -> None: - """Register *converter* and its PyArrow/Polars extension types. - - Args: - converter: An ``ExtensionTypeConverter`` instance to register. - - Raises: - ValueError: If ``converter.extension_name`` is already registered - in this registry instance. - ValueError: If the extension name is already in the PA or Polars - global registry with different parameters. - ValueError: If the extension name is already in the PA or Polars - global registry from an external source (equivalence cannot - be verified). - """ - name = converter.extension_name - if name in self._by_name: - raise ValueError( - f"Extension type '{name}' is already registered in this registry." - ) - self._by_name[name] = converter - self._by_python_type[converter.python_type] = converter - _register_arrow_ext_type(converter) - _register_polars_ext_type(converter) - - def get_converter_for_name(self, name: str) -> ExtensionTypeConverter | None: - """Return the converter registered under *name*, or ``None``.""" - return self._by_name.get(name) - - def get_converter_for_python_type(self, python_type: type) -> ExtensionTypeConverter | None: - """Return the converter for *python_type*, or ``None``. - - Checks exact match first, then falls back to an ``issubclass`` scan. - When multiple registered types are superclasses of *python_type*, the - one registered first wins (insertion-order dict, Python 3.7+). - """ - converter = self._by_python_type.get(python_type) - if converter is not None: - return converter - for registered_type, conv in self._by_python_type.items(): - if issubclass(python_type, registered_type): - return conv - return None - - def has_extension_name(self, name: str) -> bool: - """Return ``True`` if *name* is registered.""" - return name in self._by_name - - def has_python_type(self, python_type: type) -> bool: - """Return ``True`` if *python_type* (or a subclass) is registered.""" - return self.get_converter_for_python_type(python_type) is not None - - def list_extension_names(self) -> list[str]: - """Return all registered extension names in insertion order.""" - return list(self._by_name.keys()) - - def list_python_types(self) -> list[type]: - """Return all registered Python types in insertion order.""" - return list(self._by_python_type.keys()) -``` - -- [ ] **Step 4: Run the pure-Python tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -k "not arrow and not polars and not round_trip and not parquet and not module_instance" -``` - -Expected: all 11 tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/registry.py tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add ExtensionTypeRegistry with pure-Python lookup (PLT-1653)" -``` - ---- - -## Task 3: Add PyArrow global registration tests - -**Files:** -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Add the PyArrow tests** - -Append to `tests/test_extension_types/test_registry.py`: - -```python -# --------------------------------------------------------------------------- -# PyArrow global registry tests -# --------------------------------------------------------------------------- - -def test_register_populates_arrow_registry(): - """After register(), PA global registry contains the extension type.""" - conv = _make_stub() - registry = ExtensionTypeRegistry() - registry.register(conv) - - # If the name is registered, attempting to re-register it raises ArrowKeyError. - # This is the only stable public signal PyArrow provides. - class _Probe(pa.ExtensionType): - def __init__(self): - pa.ExtensionType.__init__(self, pa.large_utf8(), conv.extension_name) - def __arrow_ext_serialize__(self): - return b"" - @classmethod - def __arrow_ext_deserialize__(cls, st, se): - return cls() - - with pytest.raises(pa.lib.ArrowKeyError): - pa.register_extension_type(_Probe()) - - -def test_register_arrow_global_collision_same_params_is_idempotent(): - """A second registry instance registering the same name+params succeeds silently.""" - name = _unique_name() - conv = _make_stub(name=name, storage=pa.large_utf8(), metadata=b"cat") - - ExtensionTypeRegistry().register(conv) # first — populates _ARROW_REGISTRY - ExtensionTypeRegistry().register(conv) # second — should not raise - - -def test_register_arrow_global_collision_different_storage_raises(): - """A second registry using the same name but different storage_type raises.""" - name = _unique_name() - ExtensionTypeRegistry().register(_make_stub(name=name, storage=pa.large_utf8())) - - with pytest.raises(ValueError, match=name): - ExtensionTypeRegistry().register(_make_stub(name=name, storage=pa.large_binary())) - - -def test_register_arrow_global_collision_different_metadata_raises(): - """A second registry using the same name but different metadata raises.""" - name = _unique_name() - ExtensionTypeRegistry().register(_make_stub(name=name, metadata=b"original")) - - with pytest.raises(ValueError, match=name): - ExtensionTypeRegistry().register(_make_stub(name=name, metadata=b"different")) - - -def test_register_arrow_external_registration_raises(): - """A name registered directly with PyArrow (bypassing our registry) raises on register().""" - name = _unique_name() - - class _External(pa.ExtensionType): - def __init__(self): - pa.ExtensionType.__init__(self, pa.large_utf8(), name) - def __arrow_ext_serialize__(self): - return b"" - @classmethod - def __arrow_ext_deserialize__(cls, st, se): - return cls() - - pa.register_extension_type(_External()) # bypass our registry - - with pytest.raises(ValueError, match="external source"): - ExtensionTypeRegistry().register(_make_stub(name=name)) -``` - -- [ ] **Step 2: Run all tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: all tests pass (the PyArrow registration was already wired in Task 2). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_registry.py -git commit -m "test(extension_types): add PyArrow global registry tests (PLT-1653)" -``` - ---- - -## Task 4: Add Polars global registration tests - -**Files:** -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Add the Polars tests** - -Append to `tests/test_extension_types/test_registry.py`: - -```python -# --------------------------------------------------------------------------- -# Polars global registry tests -# --------------------------------------------------------------------------- - -def test_register_populates_polars_registry(): - """After register(), pl.from_arrow on an ext-type array yields a BaseExtension dtype.""" - conv = _make_stub(storage=pa.large_utf8()) - registry = ExtensionTypeRegistry() - registry.register(conv) - - # Build a PA extension array using the registered type. - # We need to get the registered ArrowExtType instance; the simplest way is - # to read it from _ARROW_REGISTRY shadow dict via the type's name in a PA array. - from orcapod.extension_types.registry import _ARROW_REGISTRY - assert conv.extension_name in _ARROW_REGISTRY - - # Create a storage array and cast it to the ext type to get a properly typed array. - # (The ArrowExtType class is not directly accessible from outside, but we can - # construct an array through the IPC round-trip or via the registered type.) - # Simplest: use pl.from_arrow on a storage array and check the dtype AFTER - # registering — the series dtype should be our BaseExtension subclass. - import warnings - arr = pa.array(["hello"], type=pa.large_utf8()) - # The ext type is registered, so building an array with it works. - # We access it via the _ARROW_REGISTRY which stores (storage_type, metadata). - # The actual class instance is what was registered; we verify Polars recognises it - # by checking the dtype returned from pl.from_arrow on an ext-typed array. - # Build ext array via cast on a pre-registered type instance from the module. - from orcapod.extension_types import registry as reg_mod - # Reconstruct the ArrowExtType by checking what _ARROW_REGISTRY has, then - # building a matching IPC array. Easiest: use the existing ArrowExtType class - # by catching it from PA global via unregister/re-register trick — but that's - # invasive. Instead, just verify via _POLARS_REGISTRY dict directly. - from orcapod.extension_types.registry import _POLARS_REGISTRY - assert conv.extension_name in _POLARS_REGISTRY - stored_storage, stored_meta = _POLARS_REGISTRY[conv.extension_name] - assert stored_storage == pl.String - assert stored_meta == "test.category" - - -def test_register_polars_global_collision_same_params_is_idempotent(): - """A second registry instance registering the same name+params succeeds silently.""" - name = _unique_name() - conv = _make_stub(name=name, storage=pa.large_utf8(), metadata=b"cat") - - ExtensionTypeRegistry().register(conv) - ExtensionTypeRegistry().register(conv) # should not raise - - -def test_register_polars_global_collision_different_storage_raises(): - """A second registry using the same name but different storage_type raises.""" - name = _unique_name() - ExtensionTypeRegistry().register(_make_stub(name=name, storage=pa.large_utf8())) - - with pytest.raises(ValueError, match=name): - ExtensionTypeRegistry().register(_make_stub(name=name, storage=pa.large_binary())) - - -def test_register_polars_external_registration_raises(): - """A name registered directly with Polars (bypassing our registry) raises on register().""" - name = _unique_name() - - class _ExternalPL(pl.BaseExtension): - def __init__(self): - super().__init__(name, pl.String, None) - @classmethod - def ext_from_params(cls, n, s, m): - return cls() - - # Also register in PA first so we don't hit the PA external-registration error - class _ExternalPA(pa.ExtensionType): - def __init__(self): - pa.ExtensionType.__init__(self, pa.large_utf8(), name) - def __arrow_ext_serialize__(self): - return b"" - @classmethod - def __arrow_ext_deserialize__(cls, st, se): - return cls() - - pa.register_extension_type(_ExternalPA()) - pl.register_extension_type(name, _ExternalPL) - - with pytest.raises(ValueError, match="external source"): - ExtensionTypeRegistry().register(_make_stub(name=name)) -``` - -- [ ] **Step 2: Run all tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_registry.py -git commit -m "test(extension_types): add Polars global registry tests (PLT-1653)" -``` - ---- - -## Task 5: End-to-end integration tests - -**Files:** -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Add the integration tests** - -Append to `tests/test_extension_types/test_registry.py`: - -```python -# --------------------------------------------------------------------------- -# End-to-end integration tests -# --------------------------------------------------------------------------- - -import warnings -import tempfile -import pathlib -import pyarrow.parquet as pq - - -class _Color: - """Minimal Python class used to exercise the converter contract end-to-end.""" - def __init__(self, hex_str: str) -> None: - self.hex_str = hex_str - def __eq__(self, other: object) -> bool: - return isinstance(other, _Color) and self.hex_str == other.hex_str - def __repr__(self) -> str: - return f"Color({self.hex_str!r})" - - -def _make_color_converter() -> ExtensionTypeConverter: - """ExtensionTypeConverter for _Color, backed by pa.large_utf8() storage.""" - _name = _unique_name() - - class _ColorConverter: - @property - def extension_name(self) -> str: - return _name - @property - def extension_metadata(self) -> bytes | None: - return b"test.color" - @property - def storage_type(self) -> pa.DataType: - return pa.large_utf8() - @property - def python_type(self) -> type: - return _Color - def python_to_storage(self, value: _Color) -> str: - return value.hex_str - def storage_to_python(self, storage_value: str) -> _Color: - return _Color(storage_value) - - return _ColorConverter() - - -def _build_ext_array( - converter: ExtensionTypeConverter, - values: list, -) -> pa.Array: - """Build a PA extension array from Python values using the converter.""" - from orcapod.extension_types.registry import _ARROW_REGISTRY - - storage_values = [converter.python_to_storage(v) for v in values] - storage_arr = pa.array(storage_values, type=converter.storage_type) - - # Retrieve the registered ArrowExtType instance via a fresh array cast. - # We use the PA global registry indirectly: _ARROW_REGISTRY confirms - # the type is registered; we then reconstruct the ext array by building - # a new subclass instance (same extension_name → PA resolves to registered class). - import re - _name = converter.extension_name - _storage = converter.storage_type - _metadata = converter.extension_metadata or b"" - _sanitized = re.sub(r"[^A-Za-z0-9]", "_", _name) - - ArrowExtType = type( - f"_ArrowExt_{_sanitized}_probe", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _storage, _name), - "__arrow_ext_serialize__": lambda self: _metadata, - "__arrow_ext_deserialize__": classmethod(lambda cls, st, se: cls()), - }, - ) - # This will be caught as "already registered" internally; we instantiate - # separately — PyArrow resolves the extension by name, not by class identity. - ext_type_instance = ArrowExtType() - return storage_arr.cast(ext_type_instance) - - -def test_python_class_round_trip(): - """Python objects → Arrow extension array → Python objects via converter methods.""" - conv = _make_color_converter() - registry = ExtensionTypeRegistry() - registry.register(conv) - - originals = [_Color("#ff0000"), _Color("#00ff00"), _Color("#0000ff")] - ext_arr = _build_ext_array(conv, originals) - - # Decode back - storage_back = ext_arr.cast(conv.storage_type) - recovered = [conv.storage_to_python(v.as_py()) for v in storage_back] - assert recovered == originals - - -def test_arrow_polars_round_trip(): - """PA ext array → pl.from_arrow → to_arrow() preserves extension type and values.""" - conv = _make_color_converter() - registry = ExtensionTypeRegistry() - registry.register(conv) - - originals = [_Color("#aabbcc"), _Color("#112233")] - ext_arr = _build_ext_array(conv, originals) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - pl_series = pl.from_arrow(ext_arr) - - assert isinstance(pl_series.dtype, pl.BaseExtension) - assert pl_series.dtype.ext_name() == conv.extension_name - - arr_back = pl_series.to_arrow() - assert arr_back.type.extension_name == conv.extension_name - - recovered = [conv.storage_to_python(v.as_py()) for v in arr_back.cast(conv.storage_type)] - assert recovered == originals - - -def test_parquet_round_trip(): - """PA ext array → Parquet → read back via PyArrow; extension type and values preserved.""" - conv = _make_color_converter() - registry = ExtensionTypeRegistry() - registry.register(conv) - - originals = [_Color("#deadbe"), _Color("#cafeba")] - ext_arr = _build_ext_array(conv, originals) - schema = pa.schema([pa.field("color", ext_arr.type), pa.field("id", pa.int32())]) - table = pa.table( - {"color": ext_arr, "id": pa.array([1, 2], type=pa.int32())}, - schema=schema, - ) - - with tempfile.TemporaryDirectory() as tmp: - path = pathlib.Path(tmp) / "test.parquet" - pq.write_table(table, path) - table_back = pq.read_table(path) - - assert table_back.schema.field("color").type.extension_name == conv.extension_name - recovered = [ - conv.storage_to_python(v.as_py()) - for v in table_back.column("color").cast(conv.storage_type) - ] - assert recovered == originals -``` - -- [ ] **Step 2: Run all tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_registry.py -git commit -m "test(extension_types): add end-to-end integration tests (PLT-1653)" -``` - ---- - -## Task 6: Update `extension_types/__init__.py` - -**Files:** -- Modify: `tests/test_extension_types/test_registry.py` -- Modify: `src/orcapod/extension_types/__init__.py` - -- [ ] **Step 1: Write the failing test** - -Append to `tests/test_extension_types/test_registry.py`: - -```python -# --------------------------------------------------------------------------- -# Module-level instance test -# --------------------------------------------------------------------------- - -def test_extension_type_registry_module_instance(): - """extension_types.extension_type_registry is an ExtensionTypeRegistry, starts empty.""" - from orcapod import extension_types - assert isinstance(extension_types.extension_type_registry, ExtensionTypeRegistry) - # PLT-1653 scope: no built-in converters registered yet (that is PLT-1656) - assert extension_types.extension_type_registry.list_extension_names() == [] -``` - -- [ ] **Step 2: Run to confirm it fails** - -```bash -uv run pytest tests/test_extension_types/test_registry.py::test_extension_type_registry_module_instance -v -``` - -Expected: `AttributeError: module 'orcapod.extension_types' has no attribute 'extension_type_registry'` - -- [ ] **Step 3: Update `src/orcapod/extension_types/__init__.py`** - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for converters that map -between Python objects and their Arrow extension type storage representation. - -The module-level ``extension_type_registry`` instance is the process default. -Built-in registrations (``Path``, ``UPath``, ``UUID``) are added by PLT-1656. -``DataContext`` wiring is added by PLT-1660. -""" - -from .protocols import ExtensionTypeConverter -from .registry import ExtensionTypeRegistry - -extension_type_registry = ExtensionTypeRegistry() - -__all__ = [ - "ExtensionTypeConverter", - "ExtensionTypeRegistry", - "extension_type_registry", -] -``` - -- [ ] **Step 4: Run all tests** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Run the full test suite to check for regressions** - -```bash -uv run pytest --tb=short -q -``` - -Expected: no new failures. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): export ExtensionTypeRegistry and module-level instance (PLT-1653)" -``` - ---- - -## Final check - -```bash -uv run pytest tests/test_extension_types/ -v --tb=short -``` - -All tests should pass. The PR targets `dev`. diff --git a/superpowers/plans/2026-06-14-plt-1654-schema-walker.md b/superpowers/plans/2026-06-14-plt-1654-schema-walker.md deleted file mode 100644 index 1e4c25e1b..000000000 --- a/superpowers/plans/2026-06-14-plt-1654-schema-walker.md +++ /dev/null @@ -1,660 +0,0 @@ -# PLT-1654: Schema Walker Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `src/orcapod/extension_types/schema_walker.py` — a pure discovery utility that walks a `pa.Schema` or `pa.Field` recursively and returns all extension-typed fields as `ExtensionTypeInfo` instances. - -**Architecture:** Three-layer design: `ExtensionTypeInfo` frozen dataclass as the return type; `_detect_extension` handles single-field two-channel detection; `_collect` drives recursive container descent with inline deduplication. Two public entry points — `walk_schema` and `walk_field` — each initialise a fresh `seen` set and delegate to `_collect`. - -**Tech Stack:** PyArrow ≥ 20.0.0, Python 3.11+, pytest, uv - ---- - -## File Map - -| File | Change | -|---|---| -| `src/orcapod/extension_types/schema_walker.py` | **New** — full module | -| `src/orcapod/extension_types/__init__.py` | Additive — append three new exports | -| `tests/test_extension_types/test_schema_walker.py` | **New** — full test suite | - -No other files are touched. - ---- - -## Task 1: Core module — `ExtensionTypeInfo`, detection, top-level walk, deduplication - -This task produces the full `schema_walker.py`. Container recursion (struct/list/map) is -added in Task 2. After this task, `walk_schema` and `walk_field` work for top-level -fields only; nesting tests are left for Task 2. - -**Files:** -- Create: `src/orcapod/extension_types/schema_walker.py` -- Create: `tests/test_extension_types/test_schema_walker.py` - ---- - -- [ ] **Step 1.1: Write the failing tests** - -Create `tests/test_extension_types/test_schema_walker.py` with this content: - -```python -"""Tests for schema_walker — recursive Arrow extension type discovery.""" - -from __future__ import annotations - -import re -import uuid - -import pyarrow as pa -import pytest - -from orcapod.extension_types.schema_walker import ( - ExtensionTypeInfo, - walk_field, - walk_schema, -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _unique_name() -> str: - """Return a unique extension name to avoid cross-test collisions.""" - return f"test.walker.{uuid.uuid4().hex[:8]}" - - -def _make_reg_field( - field_name: str, - ext_name: str, - storage: pa.DataType | None = None, - metadata: bytes = b"test.cat", -) -> pa.Field: - """Create a ``pa.Field`` with an in-memory ``pa.ExtensionType`` (registered channel). - - The extension type is NOT registered in PyArrow's global registry — this - is intentional. ``pa.types.is_extension(field.type)`` returns ``True`` - for any ``pa.ExtensionType`` instance regardless of global registration. - """ - _n = ext_name - _s = storage if storage is not None else pa.large_utf8() - _m = metadata - ExtType = type( - f"_RegExt_{re.sub(r'[^A-Za-z0-9]', '_', ext_name)}", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _s, _n), - "__arrow_ext_serialize__": lambda self: _m, - "__arrow_ext_deserialize__": classmethod(lambda cls, st, se: cls()), - }, - ) - return pa.field(field_name, ExtType()) - - -def _make_unreg_field( - field_name: str, - ext_name: str, - storage: pa.DataType | None = None, - metadata: bytes = b"test.cat", -) -> pa.Field: - """Create a ``pa.Field`` with raw Arrow extension metadata (unregistered channel).""" - _s = storage if storage is not None else pa.large_utf8() - return pa.field( - field_name, - _s, - metadata={ - b"ARROW:extension:name": ext_name.encode(), - b"ARROW:extension:metadata": metadata, - }, - ) - - -# --------------------------------------------------------------------------- -# Task 1 tests: top-level detection and deduplication -# --------------------------------------------------------------------------- - - -def test_empty_schema(): - result = walk_schema(pa.schema([])) - assert result == [] - - -def test_no_extension_types(): - schema = pa.schema([ - pa.field("x", pa.int64()), - pa.field("y", pa.large_utf8()), - ]) - assert walk_schema(schema) == [] - - -def test_top_level_registered(): - name = _unique_name() - schema = pa.schema([_make_reg_field("col", name, metadata=b"my.cat")]) - result = walk_schema(schema) - assert len(result) == 1 - assert result[0].extension_name == name - assert result[0].extension_metadata == b"my.cat" - assert result[0].storage_type == pa.large_utf8() - - -def test_top_level_unregistered(): - name = _unique_name() - schema = pa.schema([_make_unreg_field("col", name, metadata=b"my.cat")]) - result = walk_schema(schema) - assert len(result) == 1 - assert result[0].extension_name == name - assert result[0].extension_metadata == b"my.cat" - assert result[0].storage_type == pa.large_utf8() - - -def test_empty_metadata_normalised_to_none_registered(): - """b'' from __arrow_ext_serialize__ is normalised to None.""" - name = _unique_name() - _n, _s = name, pa.large_utf8() - ExtType = type( - "_EmptyMetaExt", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _s, _n), - "__arrow_ext_serialize__": lambda self: b"", - "__arrow_ext_deserialize__": classmethod(lambda cls, st, se: cls()), - }, - ) - result = walk_field(pa.field("col", ExtType())) - assert len(result) == 1 - assert result[0].extension_metadata is None - - -def test_empty_metadata_normalised_to_none_unregistered(): - """b'' ARROW:extension:metadata value is normalised to None.""" - name = _unique_name() - field = pa.field( - "col", - pa.large_utf8(), - metadata={ - b"ARROW:extension:name": name.encode(), - b"ARROW:extension:metadata": b"", - }, - ) - result = walk_field(field) - assert len(result) == 1 - assert result[0].extension_metadata is None - - -def test_walk_field_returns_single_field_result(): - name = _unique_name() - field = _make_reg_field("col", name, metadata=b"cat") - result = walk_field(field) - assert len(result) == 1 - assert result[0].extension_name == name - - -def test_deduplication(): - """Same (extension_name, extension_metadata) in two columns → one result.""" - name = _unique_name() - meta = b"test.cat" - _n, _m, _s = name, meta, pa.large_utf8() - ExtType = type( - "_DupExt", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _s, _n), - "__arrow_ext_serialize__": lambda self: _m, - "__arrow_ext_deserialize__": classmethod(lambda cls, st, se: cls()), - }, - ) - schema = pa.schema([ - pa.field("col_a", ExtType()), - pa.field("col_b", ExtType()), - ]) - result = walk_schema(schema) - assert len(result) == 1 - assert result[0].extension_name == name - assert result[0].extension_metadata == meta -``` - -- [ ] **Step 1.2: Run tests to verify they all fail** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_extension_types/test_schema_walker.py -v 2>&1 | head -30 -``` - -Expected: `ModuleNotFoundError` or `ImportError` — `schema_walker` does not exist yet. - -- [ ] **Step 1.3: Implement `schema_walker.py`** - -Create `src/orcapod/extension_types/schema_walker.py` with this content: - -```python -"""Recursive Arrow schema walker for extension type discovery. - -Given a ``pa.Schema`` or a single ``pa.Field``, walks the Arrow type tree -recursively and returns all extension-typed fields found at any depth of -nesting (struct, list, map, etc.). - -This is a pure discovery utility — it never triggers any registration. -""" - -from __future__ import annotations - -import dataclasses - -import pyarrow as pa - - -@dataclasses.dataclass(frozen=True) -class ExtensionTypeInfo: - """Metadata for a single Arrow extension type found in a schema. - - Attributes: - extension_name: The extension type's unique name stored as - ``ARROW:extension:name`` (e.g. ``"pathlib.Path"``). - extension_metadata: The category tag stored as - ``ARROW:extension:metadata`` (e.g. ``b"orcapod.dataclass"``). - ``None`` when absent or serialised as empty bytes. - storage_type: The underlying Arrow storage type - (e.g. ``pa.large_string()``). - """ - - extension_name: str - extension_metadata: bytes | None - storage_type: pa.DataType - - -def walk_schema(schema: pa.Schema) -> list[ExtensionTypeInfo]: - """Walk *schema* and return all extension types found, deduplicated. - - Iterates every top-level field and descends recursively into struct, - list, and map container types. The result is deduplicated by - ``(extension_name, extension_metadata)``; the first occurrence of each - pair is kept. - - Args: - schema: A PyArrow schema to inspect. - - Returns: - Deduplicated list of ``ExtensionTypeInfo`` in depth-first, - first-seen order. - """ - seen: set[tuple[str, bytes | None]] = set() - results: list[ExtensionTypeInfo] = [] - for i in range(schema.num_fields): - _collect(schema.field(i), seen, results) - return results - - -def walk_field(field: pa.Field) -> list[ExtensionTypeInfo]: - """Walk *field*'s type tree and return all extension types found, deduplicated. - - Args: - field: A PyArrow field to inspect. - - Returns: - Deduplicated list of ``ExtensionTypeInfo`` in depth-first, - first-seen order. - """ - seen: set[tuple[str, bytes | None]] = set() - results: list[ExtensionTypeInfo] = [] - _collect(field, seen, results) - return results - - -def _collect( - field: pa.Field, - seen: set[tuple[str, bytes | None]], - results: list[ExtensionTypeInfo], -) -> None: - """Recursively walk *field* and accumulate ``ExtensionTypeInfo`` into *results*. - - Mutates *seen* and *results* in place. Stops descending once a field is - identified as extension-typed — the storage type of an extension type is - not descended into. - - Args: - field: The field to inspect. - seen: Deduplication set of ``(extension_name, extension_metadata)`` - pairs already appended to *results*. - results: Accumulator list. - """ - info = _detect_extension(field) - if info is not None: - key = (info.extension_name, info.extension_metadata) - if key not in seen: - seen.add(key) - results.append(info) - return - - t = field.type - if pa.types.is_struct(t): - for i in range(t.num_fields): - _collect(t.field(i), seen, results) - elif ( - pa.types.is_list(t) - or pa.types.is_large_list(t) - or pa.types.is_fixed_size_list(t) - or pa.types.is_list_view(t) - or pa.types.is_large_list_view(t) - ): - _collect(t.value_field, seen, results) - elif pa.types.is_map(t): - key_field = getattr(t, "key_field", None) - item_field = getattr(t, "item_field", None) - if key_field is not None: - _collect(key_field, seen, results) - if item_field is not None: - _collect(item_field, seen, results) - - -def _detect_extension(field: pa.Field) -> ExtensionTypeInfo | None: - """Extract ``ExtensionTypeInfo`` from *field*, or ``None`` if not extension-typed. - - Checks two channels in order: - - 1. **Registered channel** — ``pa.types.is_extension(field.type)`` is - true. The Python type object carries the name, serialised metadata, - and storage type. - 2. **Unregistered channel** — ``field.metadata`` contains - ``b"ARROW:extension:name"``. The type survived a Parquet/IPC - round-trip without being registered in this process. - - In both cases empty bytes metadata (``b""``) is normalised to ``None``. - - Args: - field: The field to inspect. - - Returns: - ``ExtensionTypeInfo`` if the field is extension-typed, else ``None``. - """ - if pa.types.is_extension(field.type): - ext_type = field.type - raw_meta = ext_type.__arrow_ext_serialize__() - return ExtensionTypeInfo( - extension_name=ext_type.extension_name, - extension_metadata=raw_meta or None, - storage_type=ext_type.storage_type, - ) - - if field.metadata and b"ARROW:extension:name" in field.metadata: - name = field.metadata[b"ARROW:extension:name"].decode("utf-8") - raw_meta = field.metadata.get(b"ARROW:extension:metadata") - return ExtensionTypeInfo( - extension_name=name, - extension_metadata=raw_meta or None, - storage_type=field.type, - ) - - return None -``` - -- [ ] **Step 1.4: Run Task 1 tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_schema_walker.py -v -k "empty_schema or no_extension or top_level or empty_metadata or walk_field or deduplication" -``` - -Expected: all 8 tests PASS. - -- [ ] **Step 1.5: Commit** - -```bash -git add src/orcapod/extension_types/schema_walker.py tests/test_extension_types/test_schema_walker.py -git commit -m "feat(extension_types): add schema_walker with ExtensionTypeInfo and top-level detection (PLT-1654)" -``` - ---- - -## Task 2: Container recursion — struct, list, map, nested combinations - -This task adds the nesting tests and verifies the container recursion already present in -`_collect` (written in Task 1) handles them correctly. - -**Files:** -- Modify: `tests/test_extension_types/test_schema_walker.py` — append new tests - ---- - -- [ ] **Step 2.1: Append the nesting tests** - -Append to `tests/test_extension_types/test_schema_walker.py`: - -```python -# --------------------------------------------------------------------------- -# Task 2 tests: container recursion -# --------------------------------------------------------------------------- - - -def test_list_of_registered(): - """Registered extension type as the value field of a list.""" - name = _unique_name() - value_field = _make_reg_field("item", name, metadata=b"my.cat") - list_field = pa.field("col", pa.list_(value_field)) - result = walk_schema(pa.schema([list_field])) - assert len(result) == 1 - assert result[0].extension_name == name - - -def test_list_of_unregistered(): - """Unregistered extension type as the value field of a list.""" - name = _unique_name() - value_field = _make_unreg_field("item", name, metadata=b"my.cat") - list_field = pa.field("col", pa.list_(value_field)) - result = walk_schema(pa.schema([list_field])) - assert len(result) == 1 - assert result[0].extension_name == name - assert result[0].extension_metadata == b"my.cat" - - -def test_struct_containing_registered(): - """Registered extension type as a field inside a struct.""" - name = _unique_name() - struct_field = pa.field( - "col", - pa.struct([ - _make_reg_field("a", name, metadata=b"my.cat"), - pa.field("b", pa.int64()), - ]), - ) - result = walk_schema(pa.schema([struct_field])) - assert len(result) == 1 - assert result[0].extension_name == name - - -def test_struct_containing_unregistered(): - """Unregistered extension type as a field inside a struct.""" - name = _unique_name() - struct_field = pa.field( - "col", - pa.struct([ - _make_unreg_field("a", name, metadata=b"my.cat"), - pa.field("b", pa.int64()), - ]), - ) - result = walk_schema(pa.schema([struct_field])) - assert len(result) == 1 - assert result[0].extension_name == name - assert result[0].extension_metadata == b"my.cat" - - -def test_nested_list_struct(): - """Registered extension type nested inside list>.""" - name = _unique_name() - struct_type = pa.struct([ - _make_reg_field("x", name, metadata=b"deep.cat"), - pa.field("y", pa.int32()), - ]) - value_field = pa.field("item", struct_type) - col = pa.field("col", pa.list_(value_field)) - result = walk_schema(pa.schema([col])) - assert len(result) == 1 - assert result[0].extension_name == name - assert result[0].extension_metadata == b"deep.cat" - - -def test_map_type(): - """Extension type as the item type of a map (registered channel).""" - name = _unique_name() - _n, _m, _s = name, b"map.cat", pa.large_utf8() - # Build a pa.ExtensionType instance — it IS a pa.DataType and can be - # passed directly to pa.map_() as the item type. - ExtType = type( - "_MapItemExt", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _s, _n), - "__arrow_ext_serialize__": lambda self: _m, - "__arrow_ext_deserialize__": classmethod(lambda cls, st, se: cls()), - }, - ) - map_field = pa.field("col", pa.map_(pa.large_utf8(), ExtType())) - result = walk_schema(pa.schema([map_field])) - # _collect uses getattr(t, "item_field") to retrieve the item pa.Field. - # pa.types.is_extension(item_field.type) will be True for the ExtType above. - assert any(r.extension_name == name for r in result) -``` - -- [ ] **Step 2.2: Run the nesting tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_schema_walker.py -v -k "list_of or struct_containing or nested or map_type" -``` - -Expected: all 6 tests PASS. The recursion was already written in `_collect` in Task 1. - -If `test_map_type` fails because `key_field` / `item_field` are not available on `MapType` -in this PyArrow version, skip it with `@pytest.mark.skip` and open a follow-up note. - -- [ ] **Step 2.3: Run the full test file to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_schema_walker.py -v -``` - -Expected: all 14 tests PASS. - -- [ ] **Step 2.4: Commit** - -```bash -git add tests/test_extension_types/test_schema_walker.py -git commit -m "test(extension_types): add nesting and map tests for schema_walker (PLT-1654)" -``` - ---- - -## Task 3: Export from `__init__.py` - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` - ---- - -- [ ] **Step 3.1: Update `__init__.py`** - -Open `src/orcapod/extension_types/__init__.py`. It currently reads: - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for converters that map -between Python objects and their Arrow extension type storage representation. - -The module-level `default_extension_type_registry` instance is the process default. -Built-in registrations (`Path`, `UPath`, `UUID`) are added by PLT-1656. -`DataContext` wiring is added by PLT-1660. -""" - -from __future__ import annotations - -from .protocols import ExtensionTypeConverter -from .registry import ExtensionTypeRegistry - -default_extension_type_registry = ExtensionTypeRegistry() - -__all__ = [ - "ExtensionTypeConverter", - "ExtensionTypeRegistry", - "default_extension_type_registry", -] -``` - -Replace the entire file with: - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for converters that map -between Python objects and their Arrow extension type storage representation. - -The module-level `default_extension_type_registry` instance is the process default. -Built-in registrations (`Path`, `UPath`, `UUID`) are added by PLT-1656. -`DataContext` wiring is added by PLT-1660. -""" - -from __future__ import annotations - -from .protocols import ExtensionTypeConverter -from .registry import ExtensionTypeRegistry -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema - -default_extension_type_registry = ExtensionTypeRegistry() - -__all__ = [ - "ExtensionTypeConverter", - "ExtensionTypeRegistry", - "default_extension_type_registry", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", -] -``` - -- [ ] **Step 3.2: Verify the exports are importable** - -```bash -uv run python -c " -from orcapod.extension_types import ExtensionTypeInfo, walk_schema, walk_field -import pyarrow as pa -schema = pa.schema([pa.field('x', pa.int64())]) -print(walk_schema(schema)) # should print [] -print('OK') -" -``` - -Expected output: -``` -[] -OK -``` - -- [ ] **Step 3.3: Run the full test suite for `test_extension_types/`** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all tests PASS (no regressions in `test_protocols.py` or `test_registry.py`). - -- [ ] **Step 3.4: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "feat(extension_types): export ExtensionTypeInfo, walk_schema, walk_field (PLT-1654)" -``` - ---- - -## Done - -After Task 3: - -- `src/orcapod/extension_types/schema_walker.py` is complete with `ExtensionTypeInfo`, - `walk_schema`, `walk_field`, `_collect`, and `_detect_extension`. -- `ExtensionTypeInfo`, `walk_schema`, `walk_field` are exported from - `orcapod.extension_types`. -- 14 tests in `tests/test_extension_types/test_schema_walker.py` all pass. -- No existing code was modified; no regressions in other `test_extension_types/` tests. - -Create a PR targeting the `extension-type-system` branch (not `dev`). diff --git a/superpowers/plans/2026-06-14-plt-1655-database-hooks.md b/superpowers/plans/2026-06-14-plt-1655-database-hooks.md deleted file mode 100644 index e1f8f85ac..000000000 --- a/superpowers/plans/2026-06-14-plt-1655-database-hooks.md +++ /dev/null @@ -1,1300 +0,0 @@ -# PLT-1655: Database Hooks Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the peek-schema → register → read pattern to both database read paths so that Arrow extension types found in stored schemas are automatically registered before any table data is returned. - -**Architecture:** A stateless `ensure_extensions_registered(schema)` hook in `database_hooks.py` walks the schema using the existing `walk_schema` utility, then delegates each discovered type to `LogicalTypeRegistry.prepare_extension_type`. The registry owns all dispatch logic: it checks its own `_by_arrow_name` dict as a per-process cache (step 1), then parses JSON metadata, dispatches to a `LogicalTypeFactory` by category string, and calls `self.register()`. Two new protocols (`LogicalTypeFactory`) and two new methods on `LogicalTypeRegistry` (`register_logical_type_factory`, `prepare_extension_type`) complete the contract. - -**Tech Stack:** Python 3.12+, PyArrow, Polars, `pytest`, `json` (stdlib) - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/extension_types/protocols.py` | Modify | Add `LogicalTypeFactory` Protocol | -| `src/orcapod/extension_types/registry.py` | Modify | Add logging, `import json`, `LogicalTypeFactory` import, `_factories` dict, `register_logical_type_factory`, `prepare_extension_type`, move `default_logical_type_registry` singleton here | -| `src/orcapod/extension_types/__init__.py` | Modify | Import `default_logical_type_registry` from `.registry`, add `LogicalTypeFactory` and `ensure_extensions_registered` to exports | -| `src/orcapod/extension_types/database_hooks.py` | **Create** | `ensure_extensions_registered(schema)` stateless hook | -| `src/orcapod/databases/delta_lake_databases.py` | Modify | Add `ensure_extensions_registered` call in `_read_delta_table` | -| `src/orcapod/databases/connector_arrow_database.py` | Modify | Add `import logging`, `logger`, `ensure_extensions_registered` call in `_get_committed_table` | -| `tests/test_extension_types/test_protocols.py` | Modify | Add `LogicalTypeFactory` conformance tests | -| `tests/test_extension_types/test_registry.py` | Modify | Add `_make_stub_factory` helper + 9 tests for new registry methods | -| `tests/test_extension_types/test_database_hooks.py` | **Create** | 9 tests for `ensure_extensions_registered` | - ---- - -## Task 1: `LogicalTypeFactory` Protocol + registry logging infrastructure - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py` -- Modify: `src/orcapod/extension_types/registry.py` (lines 1–21: imports section) -- Test: `tests/test_extension_types/test_protocols.py` - -- [ ] **Step 1: Write the failing tests** - -Add to `tests/test_extension_types/test_protocols.py` — after the existing `_StubLogicalType` class: - -```python -class _StubFactory: - """Minimal conforming implementation of LogicalTypeFactory for use in tests.""" - - def create_logical_type(self, arrow_extension_name, storage_type, metadata): - return _StubLogicalType() - - -def test_logical_type_factory_protocol_is_importable(): - """LogicalTypeFactory can be imported from extension_types.protocols.""" - from orcapod.extension_types.protocols import LogicalTypeFactory - assert LogicalTypeFactory is not None - - -def test_logical_type_factory_conforming_class_satisfies_protocol(): - """A conforming class is recognized as a LogicalTypeFactory instance.""" - from orcapod.extension_types.protocols import LogicalTypeFactory - assert isinstance(_StubFactory(), LogicalTypeFactory) - - -def test_logical_type_factory_create_returns_logical_type(): - """A conforming factory returns a LogicalType from create_logical_type.""" - from orcapod.extension_types.protocols import LogicalTypeFactory, LogicalType - factory: LogicalTypeFactory = _StubFactory() - result = factory.create_logical_type( - "test.ext", pa.large_utf8(), {"category": "Test"} - ) - assert isinstance(result, LogicalType) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -``` -uv run pytest tests/test_extension_types/test_protocols.py -v -k "factory" -``` - -Expected: FAIL — `ImportError: cannot import name 'LogicalTypeFactory' from 'orcapod.extension_types.protocols'` - -- [ ] **Step 3: Add `LogicalTypeFactory` to protocols.py** - -Open `src/orcapod/extension_types/protocols.py`. After the closing `...` of `LogicalType`, append: - -```python -@runtime_checkable -class LogicalTypeFactory(Protocol): - """Protocol for factories that auto-construct ``LogicalType`` instances from Arrow schema metadata. - - A ``LogicalTypeFactory`` constructs a ``LogicalType`` from the Arrow extension - type name, its underlying storage type, and the full parsed JSON metadata dict. - The dispatch key (``"category"`` value from the metadata JSON) that routes to this - factory is declared at registration time via - ``LogicalTypeRegistry.register_logical_type_factory``; the factory itself has no - knowledge of its dispatch key but receives the full metadata dict so it can read - additional hints beyond ``"category"``. - - This protocol is ``@runtime_checkable``, consistent with ``LogicalType``. - """ - - def create_logical_type( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict, - ) -> LogicalType: - """Construct a ``LogicalType`` for the given Arrow extension name and storage type. - - Args: - arrow_extension_name: The Arrow extension type name extracted from the - schema (i.e. the value of ``ARROW:extension:name`` field metadata). - storage_type: The underlying Arrow storage type for this extension field. - metadata: The full parsed JSON metadata dict. Always contains at least a - ``"category"`` key. May contain additional keys the factory uses (e.g. - ``"protocol"``, ``"pydantic_version"``). - - Returns: - A fully constructed ``LogicalType`` ready to be passed to - ``LogicalTypeRegistry.register()``. - - Raises: - ValueError: If this factory cannot construct a logical type for the given - extension name (e.g. the Python class cannot be resolved by name). - """ - ... -``` - -- [ ] **Step 4: Add logging infrastructure to registry.py** - -Open `src/orcapod/extension_types/registry.py`. The current imports block starts at line 1: - -```python -from __future__ import annotations - -import re -from typing import TYPE_CHECKING - -from orcapod.extension_types.protocols import LogicalType -``` - -Replace the imports block with: - -```python -from __future__ import annotations - -import json -import logging -import re -from typing import TYPE_CHECKING - -from orcapod.extension_types.protocols import LogicalType, LogicalTypeFactory -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) -``` - -- [ ] **Step 5: Run tests to verify they pass** - -``` -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: all tests PASS (including the 3 new factory tests) - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py src/orcapod/extension_types/registry.py tests/test_extension_types/test_protocols.py -git commit -m "feat(extension_types): add LogicalTypeFactory protocol and registry logging setup" -``` - ---- - -## Task 2: Move `default_logical_type_registry` singleton to registry.py - -**Context:** `database_hooks.py` (Task 5) will import `default_logical_type_registry` from `registry.py`. If it imported from `orcapod.extension_types` (the package `__init__.py`), a circular import would occur because `__init__.py` will later import from `database_hooks`. Moving the singleton to `registry.py` breaks the cycle. - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` (add singleton at bottom) -- Modify: `src/orcapod/extension_types/__init__.py` (import instead of create) -- Test: `tests/test_extension_types/test_registry.py` (add one new import-path test) - -- [ ] **Step 1: Write the new import-path test** - -Add to the bottom of `tests/test_extension_types/test_registry.py` (after the existing `default_logical_type_registry` tests): - -```python -def test_default_registry_accessible_from_registry_module(): - """default_logical_type_registry imported from registry module is same object as from package.""" - from orcapod.extension_types.registry import default_logical_type_registry as from_registry - from orcapod.extension_types import default_logical_type_registry as from_package - assert from_registry is from_package -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_extension_types/test_registry.py::test_default_registry_accessible_from_registry_module -v -``` - -Expected: FAIL — `ImportError: cannot import name 'default_logical_type_registry' from 'orcapod.extension_types.registry'` - -- [ ] **Step 3: Add singleton to the bottom of registry.py** - -Open `src/orcapod/extension_types/registry.py`. Append after the `LogicalTypeRegistry` class: - -```python -# Module-level singleton — per-process registry used by database_hooks and -# application code. Defined here (not in __init__.py) to avoid the circular -# import that would arise if database_hooks imported from the package __init__. -default_logical_type_registry = LogicalTypeRegistry() -``` - -- [ ] **Step 4: Update __init__.py to import singleton from registry** - -Open `src/orcapod/extension_types/__init__.py`. The current content is: - -```python -"""Arrow/Polars extension type system for orcapod. -... -""" - -from __future__ import annotations - -from .protocols import LogicalType -from .registry import LogicalTypeRegistry, make_arrow_extension_type -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema - -default_logical_type_registry = LogicalTypeRegistry() - -__all__ = [ - "LogicalType", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "default_logical_type_registry", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", -] -``` - -Replace with: - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for logical types that map -between Python objects and their Arrow/Polars extension type representation. - -The module-level ``default_logical_type_registry`` instance is the process default. -Built-in registrations (``Path``, ``UPath``, ``UUID``) are added by PLT-1656. -``DataContext`` wiring is added by PLT-1660. -""" - -from __future__ import annotations - -from .protocols import LogicalType, LogicalTypeFactory -from .registry import LogicalTypeRegistry, make_arrow_extension_type, default_logical_type_registry -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema - -__all__ = [ - "LogicalType", - "LogicalTypeFactory", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "default_logical_type_registry", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", -] -``` - -- [ ] **Step 5: Run all extension_types tests to verify no regressions** - -``` -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all existing tests PASS including the new `test_default_registry_accessible_from_registry_module` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/registry.py src/orcapod/extension_types/__init__.py tests/test_extension_types/test_registry.py -git commit -m "refactor(extension_types): move default_logical_type_registry singleton to registry.py" -``` - ---- - -## Task 3: `_factories` dict + `register_logical_type_factory` method - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` -- Test: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Write the failing tests** - -Add to `tests/test_extension_types/test_registry.py` — after the `_make_stub` helper: - -```python -def _make_stub_factory(return_lt: LogicalType | None = None) -> LogicalTypeFactory: - """Factory for minimal LogicalTypeFactory conforming stubs. - - If ``return_lt`` is given, ``create_logical_type`` returns it; otherwise - it creates a fresh stub using ``_make_stub`` keyed on the arrow name. - ``calls`` records every invocation as ``(arrow_extension_name, storage_type, metadata)``. - """ - from orcapod.extension_types.protocols import LogicalTypeFactory - _return_lt = return_lt - - class _Factory: - def __init__(self): - self.calls: list[tuple] = [] - - def create_logical_type(self, arrow_extension_name, storage_type, metadata): - self.calls.append((arrow_extension_name, storage_type, metadata)) - if _return_lt is not None: - return _return_lt - return _make_stub(arrow_name=arrow_extension_name, storage=storage_type) - - return _Factory() -``` - -Then add these tests (before the `# end-to-end integration tests` section): - -```python -# --------------------------------------------------------------------------- -# register_logical_type_factory tests -# --------------------------------------------------------------------------- - -def test_register_logical_type_factory_no_error(): - """register_logical_type_factory completes without raising.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory("TestCat", factory) # should not raise - - -def test_register_logical_type_factory_same_instance_idempotent(): - """Re-registering the same factory instance for the same category does not raise.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory("Cat", factory) - registry.register_logical_type_factory("Cat", factory) # should not raise - - -def test_register_duplicate_category_raises(): - """Registering a different factory for an already-registered category raises ValueError.""" - registry = LogicalTypeRegistry() - f1 = _make_stub_factory() - f2 = _make_stub_factory() - registry.register_logical_type_factory("Cat", f1) - with pytest.raises(ValueError, match="Cat"): - registry.register_logical_type_factory("Cat", f2) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -``` -uv run pytest tests/test_extension_types/test_registry.py -v -k "factory" --no-header 2>&1 | tail -20 -``` - -Expected: FAIL — `AttributeError: 'LogicalTypeRegistry' object has no attribute 'register_logical_type_factory'` - -- [ ] **Step 3: Add `_factories` dict and `register_logical_type_factory` to LogicalTypeRegistry** - -In `src/orcapod/extension_types/registry.py`, inside the `LogicalTypeRegistry` class, update `__init__`: - -```python - def __init__(self) -> None: - self._by_logical_name: dict[str, LogicalType] = {} - self._by_arrow_name: dict[str, LogicalType] = {} - self._by_python_type: dict[type, LogicalType] = {} - self._factories: dict[str, LogicalTypeFactory] = {} -``` - -Then add the new method after `get_by_arrow_extension_name`: - -```python - def register_logical_type_factory( - self, - category: str, - factory: LogicalTypeFactory, - ) -> None: - """Register a factory for the given metadata category string. - - When ``prepare_extension_type`` encounters an Arrow extension type whose - ``extension_metadata`` JSON contains ``{"category": "", ...}``, - it calls ``factory.create_logical_type(arrow_extension_name, storage_type, - metadata_dict)`` to construct the logical type and then registers it. - - Args: - category: The ``"category"`` value from the extension metadata JSON that - identifies this category (e.g. ``"Dataclass"``). - factory: A ``LogicalTypeFactory`` instance responsible for constructing - logical types for this category. - - Raises: - ValueError: If ``category`` is already registered to a different factory. - """ - existing = self._factories.get(category) - if existing is not None and existing is not factory: - raise ValueError( - f"Cannot register factory for category {category!r}: " - f"a different factory is already registered for this category." - ) - if existing is factory: - return - self._factories[category] = factory - logger.debug( - "registered LogicalTypeFactory for category %r: %r", category, factory - ) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -``` -uv run pytest tests/test_extension_types/test_registry.py -v -k "factory" --no-header -``` - -Expected: the 3 new factory tests PASS - -- [ ] **Step 5: Run full extension_types test suite to check for regressions** - -``` -uv run pytest tests/test_extension_types/ -v --no-header 2>&1 | tail -10 -``` - -Expected: all tests PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/registry.py tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add _factories dict and register_logical_type_factory to LogicalTypeRegistry" -``` - ---- - -## Task 4: `prepare_extension_type` — full implementation (all 7 steps) - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` -- Test: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Write ALL failing tests (happy path + error paths)** - -Add to `tests/test_extension_types/test_registry.py`, after the `register_logical_type_factory` tests. Note the `import json` needed at the top of the file: - -First add `import json` to the existing import block at the top of test_registry.py (after `import uuid`). - -Then add these tests: - -```python -# --------------------------------------------------------------------------- -# prepare_extension_type tests -# --------------------------------------------------------------------------- - -def test_register_logical_type_factory_dispatches_on_prepare(): - """prepare_extension_type dispatches to the registered factory and registers the result.""" - import json - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory("TestCat", factory) - - arrow_name = _unique_name() - metadata_bytes = json.dumps({"category": "TestCat"}).encode() - registry.prepare_extension_type(arrow_name, metadata_bytes, pa.large_utf8()) - - assert len(factory.calls) == 1 - assert factory.calls[0][0] == arrow_name - assert registry.get_by_arrow_extension_name(arrow_name) is not None - - -def test_factory_receives_full_metadata_dict(): - """The factory's create_logical_type receives the full parsed JSON dict, not just category.""" - import json - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory("TestCat", factory) - - arrow_name = _unique_name() - metadata_bytes = json.dumps( - {"category": "TestCat", "protocol": 5, "version": "1.0"} - ).encode() - registry.prepare_extension_type(arrow_name, metadata_bytes, pa.large_utf8()) - - assert len(factory.calls) == 1 - _, _, received_metadata = factory.calls[0] - assert received_metadata == {"category": "TestCat", "protocol": 5, "version": "1.0"} - - -def test_prepare_already_registered_noop(): - """prepare_extension_type called twice does not raise and does not call the factory again.""" - import json - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory("TestCat", factory) - - arrow_name = _unique_name() - metadata_bytes = json.dumps({"category": "TestCat"}).encode() - - registry.prepare_extension_type(arrow_name, metadata_bytes, pa.large_utf8()) - registry.prepare_extension_type(arrow_name, metadata_bytes, pa.large_utf8()) # second call - - assert len(factory.calls) == 1 # factory called exactly once - - -def test_prepare_already_registered_none_metadata_noop(): - """Type pre-registered via register(); None metadata on prepare call is a silent no-op.""" - registry = LogicalTypeRegistry() - lt = _make_stub() - registry.register(lt) - - arrow_name = lt.get_arrow_extension_type().extension_name - registry.prepare_extension_type(arrow_name, None, pa.large_utf8()) # should not raise - - -def test_prepare_none_metadata_not_registered_raises(): - """None metadata for an unregistered extension type raises ValueError.""" - registry = LogicalTypeRegistry() - arrow_name = _unique_name() - - with pytest.raises(ValueError, match="must be pre-registered explicitly"): - registry.prepare_extension_type(arrow_name, None, pa.large_utf8()) - - -def test_prepare_invalid_json_raises(): - """Non-UTF-8-JSON extension_metadata raises ValueError with raw bytes and parse error.""" - registry = LogicalTypeRegistry() - arrow_name = _unique_name() - bad_metadata = b"not-json!" - - with pytest.raises(ValueError, match="not valid UTF-8 JSON"): - registry.prepare_extension_type(arrow_name, bad_metadata, pa.large_utf8()) - - -def test_prepare_json_missing_category_raises(): - """Valid JSON metadata without a 'category' key raises ValueError.""" - import json - registry = LogicalTypeRegistry() - arrow_name = _unique_name() - no_category = json.dumps({"version": 1}).encode() - - with pytest.raises(ValueError, match='"category"'): - registry.prepare_extension_type(arrow_name, no_category, pa.large_utf8()) - - -def test_prepare_unknown_category_raises(): - """Valid JSON with 'category' but no matching factory raises ValueError.""" - import json - registry = LogicalTypeRegistry() - arrow_name = _unique_name() - unknown = json.dumps({"category": "NoSuchFactory"}).encode() - - with pytest.raises(ValueError, match="NoSuchFactory"): - registry.prepare_extension_type(arrow_name, unknown, pa.large_utf8()) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -``` -uv run pytest tests/test_extension_types/test_registry.py -v -k "prepare" --no-header 2>&1 | tail -20 -``` - -Expected: FAIL — `AttributeError: 'LogicalTypeRegistry' object has no attribute 'prepare_extension_type'` - -- [ ] **Step 3: Implement `prepare_extension_type` in registry.py** - -Add this method to `LogicalTypeRegistry` (after `register_logical_type_factory`): - -```python - def prepare_extension_type( - self, - arrow_extension_name: str, - extension_metadata: bytes | None, - storage_type: pa.DataType, - ) -> None: - """Ensure the Arrow extension type identified by ``arrow_extension_name`` - is registered as a ``LogicalType``. - - This is the single entry point called by ``ensure_extensions_registered`` - in ``database_hooks``. The registry owns all dispatch logic. - - Args: - arrow_extension_name: Arrow extension type name (``ARROW:extension:name``). - extension_metadata: Raw metadata bytes (``ARROW:extension:metadata``), - expected to be UTF-8 JSON containing at least a ``"category"`` key. - ``None`` if absent. - storage_type: Underlying Arrow storage type for this extension field. - - Raises: - ValueError: If ``extension_metadata`` is ``None`` and the type is not - already registered. - ValueError: If ``extension_metadata`` is not valid UTF-8 JSON. - ValueError: If the parsed JSON has no ``"category"`` key. - ValueError: If no factory is registered for the ``"category"`` value. - ValueError: Propagated from the factory if it cannot construct a type. - """ - # Step 1: per-process cache hit — no-op regardless of metadata content. - if self.get_by_arrow_extension_name(arrow_extension_name) is not None: - logger.debug( - "prepare_extension_type: %r already registered, skipping", - arrow_extension_name, - ) - return - - # Step 2: None metadata — cannot auto-register; must be pre-registered. - if extension_metadata is None: - raise ValueError( - f"Extension type {arrow_extension_name!r} has no extension metadata " - f"(metadata is None).\n" - f"Types without a metadata category tag cannot be auto-registered via " - f"a factory — they must be pre-registered explicitly via " - f"default_logical_type_registry.register(logical_type)." - ) - - # Step 3: Parse JSON. - try: - metadata_dict = json.loads(extension_metadata.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ValueError( - f"Extension type {arrow_extension_name!r} has extension metadata that " - f"is not valid UTF-8 JSON: {extension_metadata!r}. " - f"Parse error: {exc}.\n" - f'Extension metadata must be a JSON object with at least a "category" ' - f'key, e.g. {{"category": "Dataclass"}}.' - ) from exc - - # Step 4: Require "category" key. - if "category" not in metadata_dict: - raise ValueError( - f"Extension type {arrow_extension_name!r} has extension metadata JSON " - f'with no "category" key: {metadata_dict}. Extension metadata must be ' - f'a JSON object with at least a "category" key, e.g. ' - f'{{"category": "Dataclass"}}.' - ) - - category = metadata_dict["category"] - - # Step 5: Look up factory. - factory = self._factories.get(category) - if factory is None: - raise ValueError( - f"No LogicalTypeFactory is registered for category {category!r}.\n" - f"Cannot prepare extension type {arrow_extension_name!r} for " - f"registration.\n" - f"Register a factory via " - f"default_logical_type_registry.register_logical_type_factory(\n" - f" {category!r}, factory\n" - f")." - ) - - # Step 6: Construct logical type via factory. - logger.debug( - "prepare_extension_type: %r not registered — dispatching to category %r factory", - arrow_extension_name, - category, - ) - logical_type = factory.create_logical_type( - arrow_extension_name, storage_type, metadata_dict - ) - - # Step 7: Register in all three bindings + PA/Polars global registries. - self.register(logical_type) - logger.debug( - "prepare_extension_type: successfully registered %r via %r factory", - arrow_extension_name, - category, - ) -``` - -- [ ] **Step 4: Run tests to verify they all pass** - -``` -uv run pytest tests/test_extension_types/test_registry.py -v --no-header 2>&1 | tail -15 -``` - -Expected: all tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/registry.py tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add prepare_extension_type to LogicalTypeRegistry" -``` - ---- - -## Task 5: `database_hooks.py` module + `__init__.py` exports + test suite - -**Files:** -- Create: `src/orcapod/extension_types/database_hooks.py` -- Modify: `src/orcapod/extension_types/__init__.py` -- Create: `tests/test_extension_types/test_database_hooks.py` - -- [ ] **Step 1: Write the failing test file** - -Create `tests/test_extension_types/test_database_hooks.py`: - -```python -"""Tests for ensure_extensions_registered in database_hooks.""" - -from __future__ import annotations - -import json -import uuid - -import pyarrow as pa -import pytest - -from orcapod.extension_types.registry import LogicalTypeRegistry, make_arrow_extension_type - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _unique_name() -> str: - """Unique Arrow extension name to avoid cross-test global-registry collisions.""" - return f"test.hooks.{uuid.uuid4().hex[:8]}" - - -def _make_ext_schema( - arrow_name: str, - metadata: bytes | None = None, - storage: pa.DataType | None = None, -) -> pa.Schema: - """Build a ``pa.Schema`` with one extension-typed field using ``make_arrow_extension_type``. - - Only call this when you have control over the metadata content — the resulting - field's type is an in-memory ``pa.ExtensionType`` instance, not raw field metadata. - """ - _storage = storage or pa.large_utf8() - ext_cls = make_arrow_extension_type(arrow_name, _storage, metadata=metadata) - return pa.schema([pa.field("col", ext_cls())]) - - -def _make_field_metadata_schema( - arrow_name: str, - metadata: bytes, - storage: pa.DataType | None = None, -) -> pa.Schema: - """Build a schema where the extension is described by raw Arrow field metadata. - - This simulates a Parquet/IPC read where the extension type was not registered - in the current process, so ``field.type`` is a plain Arrow storage type rather - than a ``pa.ExtensionType`` instance. - """ - _storage = storage or pa.large_utf8() - field = pa.field("col", _storage).with_metadata({ - b"ARROW:extension:name": arrow_name.encode(), - b"ARROW:extension:metadata": metadata, - }) - return pa.schema([field]) - - -def _make_stub_factory(registry: LogicalTypeRegistry): - """Return a minimal LogicalTypeFactory stub whose calls are recorded. - - The factory auto-creates a fresh ``LogicalType`` stub keyed by arrow name. - Registering this factory in *registry* causes it to also register a Polars - extension type, which requires the Arrow ext type to be in PyArrow's global - registry. To avoid cross-test collisions, each test uses a unique arrow name. - """ - class _Factory: - def __init__(self): - self.calls: list[tuple] = [] - - def create_logical_type(self, arrow_extension_name, storage_type, metadata): - import polars as pl - from orcapod.extension_types.registry import make_arrow_extension_type - - self.calls.append((arrow_extension_name, storage_type, metadata)) - - _name = arrow_extension_name - _arrow_cls = make_arrow_extension_type(_name, storage_type) - _pl_storage = pl.from_arrow(pa.array([], type=storage_type)).dtype - - class _PolarsExt(pl.BaseExtension): - def __init__(self): - super().__init__(_name, _pl_storage, None) - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - - class _StubLT: - @property - def logical_type_name(self): - return _name - @property - def python_type(self): - return str - def get_arrow_extension_type(self): - return _arrow_cls() - def get_polars_extension_type(self): - return _PolarsExt() - def python_to_storage(self, value): - return str(value) - def storage_to_python(self, storage_value): - return storage_value - - return _StubLT() - - return _Factory() - - -# --------------------------------------------------------------------------- -# Fixture -# --------------------------------------------------------------------------- - -@pytest.fixture -def fresh_registry(monkeypatch): - """A fresh LogicalTypeRegistry monkeypatched into database_hooks module.""" - import orcapod.extension_types.database_hooks as hooks - registry = LogicalTypeRegistry() - monkeypatch.setattr(hooks, "default_logical_type_registry", registry) - return registry - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - -def test_no_extension_types_is_noop(fresh_registry): - """Schema with only primitives — ensure_extensions_registered returns without touching registry.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field("name", pa.large_utf8()), - ]) - ensure_extensions_registered(schema) - # fresh_registry is empty — no error means no spurious lookup was triggered - assert fresh_registry.get_by_arrow_extension_name("anything") is None - - -def test_known_type_is_registered(fresh_registry): - """Schema with one extension type whose factory is registered — type is registered after call.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - factory = _make_stub_factory(fresh_registry) - fresh_registry.register_logical_type_factory("TestCat", factory) - - metadata_bytes = json.dumps({"category": "TestCat"}).encode() - schema = _make_ext_schema(arrow_name, metadata=metadata_bytes) - - ensure_extensions_registered(schema) - - assert fresh_registry.get_by_arrow_extension_name(arrow_name) is not None - assert len(factory.calls) == 1 - - -def test_already_registered_is_skipped(fresh_registry): - """Calling ensure_extensions_registered twice does not raise and factory is called once.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - factory = _make_stub_factory(fresh_registry) - fresh_registry.register_logical_type_factory("TestCat", factory) - - metadata_bytes = json.dumps({"category": "TestCat"}).encode() - schema = _make_ext_schema(arrow_name, metadata=metadata_bytes) - - ensure_extensions_registered(schema) - ensure_extensions_registered(schema) # second call - - assert len(factory.calls) == 1 # factory invoked exactly once - - -def test_none_metadata_already_registered_noop(fresh_registry): - """Extension type with None metadata that IS already in the registry — silent no-op.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - factory = _make_stub_factory(fresh_registry) - fresh_registry.register_logical_type_factory("TestCat", factory) - - # First: register via metadata so it ends up in the registry. - metadata_bytes = json.dumps({"category": "TestCat"}).encode() - schema_with_meta = _make_ext_schema(arrow_name, metadata=metadata_bytes) - ensure_extensions_registered(schema_with_meta) - - # Now: same arrow name but with no metadata (simulates reading the schema without - # metadata — e.g. after an IPC round-trip where the type is now registered in-process). - schema_no_meta = _make_ext_schema(arrow_name, metadata=None) # metadata=None → b"" - ensure_extensions_registered(schema_no_meta) # should NOT raise - - -def test_none_metadata_not_registered_raises(fresh_registry): - """Unregistered extension type with None metadata raises ValueError.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - schema = _make_ext_schema(arrow_name, metadata=None) # metadata=None → b"" → walker normalizes to None - - with pytest.raises(ValueError, match="must be pre-registered explicitly"): - ensure_extensions_registered(schema) - - -def test_metadata_not_json_raises(fresh_registry): - """Unregistered extension type with non-JSON metadata bytes raises ValueError.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - schema = _make_field_metadata_schema(arrow_name, metadata=b"not-json!") - - with pytest.raises(ValueError, match="not valid UTF-8 JSON"): - ensure_extensions_registered(schema) - - -def test_metadata_json_missing_category_raises(fresh_registry): - """Unregistered extension type with valid JSON but no 'category' key raises ValueError.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - schema = _make_field_metadata_schema( - arrow_name, metadata=json.dumps({"version": 1}).encode() - ) - - with pytest.raises(ValueError, match='"category"'): - ensure_extensions_registered(schema) - - -def test_unknown_metadata_raises(fresh_registry): - """Unregistered extension type with valid JSON and 'category' but no matching factory raises ValueError.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - schema = _make_field_metadata_schema( - arrow_name, metadata=json.dumps({"category": "NoSuchFactory"}).encode() - ) - - with pytest.raises(ValueError, match="NoSuchFactory"): - ensure_extensions_registered(schema) - - -def test_nested_extension_type(fresh_registry): - """Extension type inside a struct column is discovered and registered.""" - from orcapod.extension_types.database_hooks import ensure_extensions_registered - - arrow_name = _unique_name() - factory = _make_stub_factory(fresh_registry) - fresh_registry.register_logical_type_factory("TestCat", factory) - - metadata_bytes = json.dumps({"category": "TestCat"}).encode() - inner_ext_cls = make_arrow_extension_type(arrow_name, pa.large_utf8(), metadata=metadata_bytes) - - struct_type = pa.struct([pa.field("inner", inner_ext_cls())]) - schema = pa.schema([pa.field("outer", struct_type)]) - - ensure_extensions_registered(schema) - - assert fresh_registry.get_by_arrow_extension_name(arrow_name) is not None - assert len(factory.calls) == 1 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -``` -uv run pytest tests/test_extension_types/test_database_hooks.py -v --no-header 2>&1 | tail -15 -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'orcapod.extension_types.database_hooks'` - -- [ ] **Step 3: Create `database_hooks.py`** - -Create `src/orcapod/extension_types/database_hooks.py`: - -```python -"""Peek-schema hook for extension type auto-registration at database read time. - -Call ``ensure_extensions_registered(schema)`` before returning any Arrow table -from a database read path. It is a no-op when the schema contains no extension -types. -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -from orcapod.extension_types.registry import default_logical_type_registry -from orcapod.extension_types.schema_walker import walk_schema - -if TYPE_CHECKING: - import pyarrow as pa - -logger = logging.getLogger(__name__) - - -def ensure_extensions_registered(schema: pa.Schema) -> None: - """Register any extension types found in ``schema`` that are not yet known. - - Walks ``schema`` recursively to discover all Arrow extension types at any - nesting depth. For each discovered type, delegates to - ``default_logical_type_registry.prepare_extension_type``. - - Already-registered types are detected and skipped inside the registry — - this function itself is stateless. - - Args: - schema: The Arrow schema to inspect. May contain no extension types, - in which case this call is a no-op. - - Raises: - ValueError: Propagated from the registry if an extension type's metadata - has no registered factory or is malformed. - """ - found = walk_schema(schema) - if not found: - logger.debug("ensure_extensions_registered: no extension types in schema") - return - logger.debug( - "ensure_extensions_registered: found %d extension type(s) in schema: %s", - len(found), - [info.extension_name for info in found], - ) - for info in found: - default_logical_type_registry.prepare_extension_type( - info.extension_name, - info.extension_metadata, - info.storage_type, - ) -``` - -- [ ] **Step 4: Add `ensure_extensions_registered` to `__init__.py` exports** - -In `src/orcapod/extension_types/__init__.py`, add the import and export: - -```python -from .database_hooks import ensure_extensions_registered -``` - -Add `"ensure_extensions_registered"` to `__all__`. - -The final `__init__.py` should look like: - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for logical types that map -between Python objects and their Arrow/Polars extension type representation. - -The module-level ``default_logical_type_registry`` instance is the process default. -Built-in registrations (``Path``, ``UPath``, ``UUID``) are added by PLT-1656. -``DataContext`` wiring is added by PLT-1660. -""" - -from __future__ import annotations - -from .protocols import LogicalType, LogicalTypeFactory -from .registry import LogicalTypeRegistry, make_arrow_extension_type, default_logical_type_registry -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema -from .database_hooks import ensure_extensions_registered - -__all__ = [ - "LogicalType", - "LogicalTypeFactory", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "default_logical_type_registry", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", - # PLT-1655 - "ensure_extensions_registered", -] -``` - -- [ ] **Step 5: Run all tests to verify they pass** - -``` -uv run pytest tests/test_extension_types/ -v --no-header 2>&1 | tail -20 -``` - -Expected: all tests PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/database_hooks.py src/orcapod/extension_types/__init__.py tests/test_extension_types/test_database_hooks.py -git commit -m "feat(extension_types): add database_hooks.ensure_extensions_registered and update exports" -``` - ---- - -## Task 6: Hook `DeltaTableDatabase._read_delta_table` - -**Context:** `delta_lake_databases.py` already has `import logging` and `logger = logging.getLogger(__name__)`. Only a new import and a single hook call are needed. - -**Files:** -- Modify: `src/orcapod/databases/delta_lake_databases.py` - -- [ ] **Step 1: Add the import for `ensure_extensions_registered`** - -In `src/orcapod/databases/delta_lake_databases.py`, find the existing imports block. The file starts with: - -```python -from __future__ import annotations - -import logging -from collections import defaultdict -from collections.abc import Collection, Mapping -from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, cast - -from orcapod.databases.utils import coerce_record_id -from orcapod.databases.storage_utils import is_cloud_uri, parse_base_path -from orcapod.utils import arrow_utils -from orcapod.utils.lazy_module import LazyModule -``` - -Add the new import after the existing `orcapod` imports: - -```python -from orcapod.extension_types.database_hooks import ensure_extensions_registered -``` - -- [ ] **Step 2: Add the hook call in `_read_delta_table`** - -Find `_read_delta_table` (around line 818). The current code after the method docstring is: - -```python - filter_expr = None - # Use to_pyarrow_dataset with as_large_types for Polars compatible arrow table loading - dataset = delta_table.to_pyarrow_dataset(as_large_types=True) - if filters and expression is None: -``` - -Replace with (adding 2 lines after the dataset assignment): - -```python - filter_expr = None - # Use to_pyarrow_dataset with as_large_types for Polars compatible arrow table loading - dataset = delta_table.to_pyarrow_dataset(as_large_types=True) - logger.debug("_read_delta_table: peeking schema for extension type registration") - ensure_extensions_registered(delta_table.schema().to_arrow()) - if filters and expression is None: -``` - -- [ ] **Step 3: Run the full test suite** - -``` -uv run pytest tests/ -v --no-header -q 2>&1 | tail -20 -``` - -Expected: all tests PASS - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/databases/delta_lake_databases.py -git commit -m "feat(databases): call ensure_extensions_registered in DeltaTableDatabase._read_delta_table" -``` - ---- - -## Task 7: Hook `ConnectorArrowDatabase._get_committed_table` - -**Context:** `connector_arrow_database.py` currently has no `logger`. Add it alongside the hook import. - -**Files:** -- Modify: `src/orcapod/databases/connector_arrow_database.py` - -- [ ] **Step 1: Add `import logging`, `logger`, and hook import** - -In `src/orcapod/databases/connector_arrow_database.py`, the current imports block begins: - -```python -from __future__ import annotations - -import re -from collections import defaultdict -from collections.abc import Collection, Mapping -from typing import TYPE_CHECKING, Any, cast - -from orcapod.databases.utils import coerce_record_id -from orcapod.protocols.db_connector_protocol import ColumnInfo, DBConnectorProtocol -from orcapod.utils.lazy_module import LazyModule -``` - -Replace with: - -```python -from __future__ import annotations - -import logging -import re -from collections import defaultdict -from collections.abc import Collection, Mapping -from typing import TYPE_CHECKING, Any, cast - -from orcapod.databases.utils import coerce_record_id -from orcapod.extension_types.database_hooks import ensure_extensions_registered -from orcapod.protocols.db_connector_protocol import ColumnInfo, DBConnectorProtocol -from orcapod.utils.lazy_module import LazyModule - -logger = logging.getLogger(__name__) -``` - -- [ ] **Step 2: Add the hook call in `_get_committed_table`** - -Find `_get_committed_table` (around line 176). The current implementation is: - -```python - def _get_committed_table( - self, record_path: tuple[str, ...] - ) -> pa.Table | None: - """Fetch all committed records for a path from the connector.""" - table_name = self._path_to_table_name(self._path_prefix + record_path) - if table_name not in self._connector.get_table_names(): - return None - batches = list( - self._connector.iter_batches(f'SELECT * FROM "{table_name}"') - ) - if not batches: - return None - return pa.Table.from_batches(batches) -``` - -Replace with: - -```python - def _get_committed_table( - self, record_path: tuple[str, ...] - ) -> pa.Table | None: - """Fetch all committed records for a path from the connector.""" - table_name = self._path_to_table_name(self._path_prefix + record_path) - if table_name not in self._connector.get_table_names(): - return None - batches = list( - self._connector.iter_batches(f'SELECT * FROM "{table_name}"') - ) - if not batches: - return None - logger.debug("_get_committed_table: peeking schema for extension type registration") - ensure_extensions_registered(batches[0].schema) - return pa.Table.from_batches(batches) -``` - -- [ ] **Step 3: Run the full test suite** - -``` -uv run pytest tests/ -v --no-header -q 2>&1 | tail -20 -``` - -Expected: all tests PASS - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/databases/connector_arrow_database.py -git commit -m "feat(databases): add logger and ensure_extensions_registered hook to ConnectorArrowDatabase._get_committed_table" -``` - ---- - -## Final Verification - -- [ ] **Run the complete test suite one final time** - -``` -uv run pytest tests/ -q --no-header 2>&1 | tail -5 -``` - -Expected: all tests PASS, no warnings about new code - -- [ ] **Create PR targeting `extension-type-system` branch** - -```bash -gh pr create \ - --base extension-type-system \ - --title "feat(PLT-1655): add peek-schema → register → read pattern with per-process cache" \ - --body "$(cat <<'EOF' -## Summary - -* Adds `LogicalTypeFactory` Protocol — a pure factory that constructs a `LogicalType` from an Arrow extension name, storage type, and full parsed JSON metadata dict. -* Adds `register_logical_type_factory(category, factory)` and `prepare_extension_type(arrow_extension_name, metadata, storage_type)` to `LogicalTypeRegistry`. The registry's `_by_arrow_name` dict acts as the per-process cache (step 1: already-registered → immediate no-op regardless of metadata). -* Adds stateless `ensure_extensions_registered(schema)` in `extension_types/database_hooks.py`. Walks the schema, delegates each extension type to `prepare_extension_type`. -* Wires the hook into `DeltaTableDatabase._read_delta_table` (schema peek via `DeltaTable.schema().to_arrow()`) and `ConnectorArrowDatabase._get_committed_table` (schema peek via `batches[0].schema`). -* Moves `default_logical_type_registry` singleton from `__init__.py` to `registry.py` to break the circular import that would arise with `database_hooks`. -* Sufficient DEBUG-level logging throughout: discovery, cache hit, factory dispatch, successful registration. - -## Test plan - -- [ ] `uv run pytest tests/test_extension_types/ -v` — all new unit tests pass -- [ ] `uv run pytest tests/ -q` — no regressions - -Fixes PLT-1655 -EOF -)" -``` diff --git a/superpowers/plans/2026-06-14-plt-1656-builtin-logical-types.md b/superpowers/plans/2026-06-14-plt-1656-builtin-logical-types.md deleted file mode 100644 index 09c011677..000000000 --- a/superpowers/plans/2026-06-14-plt-1656-builtin-logical-types.md +++ /dev/null @@ -1,1511 +0,0 @@ -# PLT-1656: Built-in LogicalType Implementations (Path, UPath, UUID) — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement three built-in `LogicalType` classes (`LogicalPath`, `LogicalUPath`, `LogicalUUID`), wire them into `DataContext` via `v0.1.json`, and expose a `get_default_logical_type_registry()` convenience accessor. - -**Architecture:** Each `LogicalType` owns its Arrow/Polars extension type instances via class-level caching. A new `make_polars_extension_type` helper (parallel to the existing `make_arrow_extension_type`) synthesises `pl.BaseExtension` subclasses at runtime. The registry is populated via the existing `parse_objectspec` JSON object spec mechanism so `LogicalTypeRegistry` gains a `logical_types` constructor param. The module-level `default_logical_type_registry` in `extension_types/__init__.py` is removed — the canonical access path becomes `get_default_context().logical_type_registry`. - -**Tech Stack:** Python 3.12+, PyArrow ≥ 20, Polars ≥ 1.36.0, pytest, uv. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/extension_types/registry.py` | Modify | Add `make_polars_extension_type` helper; add `logical_types` param to `LogicalTypeRegistry.__init__` | -| `src/orcapod/extension_types/__init__.py` | Modify | Export `make_polars_extension_type`; remove `default_logical_type_registry` | -| `src/orcapod/extension_types/builtin_logical_types.py` | **New** | `LogicalPath`, `LogicalUPath`, `LogicalUUID` implementations | -| `src/orcapod/contexts/core.py` | Modify | Add `logical_type_registry: LogicalTypeRegistry` field to `DataContext` | -| `src/orcapod/contexts/registry.py` | Modify | Add `"logical_type_registry"` to required fields; pass it through in `_create_context_from_spec` | -| `src/orcapod/contexts/data/v0.1.json` | Modify | Add `logical_type_registry` object spec entry | -| `src/orcapod/contexts/data/schemas/context_schema.json` | Modify | Add `logical_type_registry` to `required` and `properties` | -| `src/orcapod/contexts/__init__.py` | Modify | Add `get_default_logical_type_registry()` convenience function | -| `tests/test_extension_types/test_registry.py` | Modify | Add tests for `make_polars_extension_type` and `logical_types` param; remove stale `default_logical_type_registry` tests | -| `tests/test_extension_types/test_builtin_logical_types.py` | **New** | Protocol conformance, property values, round-trips, default-context integration tests | - ---- - -### Task 1: `make_polars_extension_type` helper - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Write the failing tests** - -Add these tests at the end of `tests/test_extension_types/test_registry.py`, before the `# default_logical_type_registry tests` section: - -```python -# --------------------------------------------------------------------------- -# make_polars_extension_type tests -# --------------------------------------------------------------------------- - -from orcapod.extension_types.registry import make_polars_extension_type - - -def test_make_polars_extension_type_returns_class(): - """make_polars_extension_type returns a pl.BaseExtension subclass.""" - cls = make_polars_extension_type("test.MakePolarsExt", pa.large_utf8()) - assert issubclass(cls, pl.BaseExtension) - - -def test_make_polars_extension_type_instance_has_correct_name(): - """Instantiating the returned class yields the correct ext_name.""" - name = _unique_name() - cls = make_polars_extension_type(name, pa.large_utf8()) - inst = cls() - assert inst.ext_name() == name - - -def test_make_polars_extension_type_ext_from_params_returns_instance(): - """ext_from_params classmethod returns an instance of the class.""" - name = _unique_name() - cls = make_polars_extension_type(name, pa.large_utf8()) - inst = cls.ext_from_params(name, pl.String, None) - assert isinstance(inst, cls) - - -def test_make_polars_extension_type_with_binary_storage(): - """make_polars_extension_type works with pa.binary(16) storage (UUID case).""" - name = _unique_name() - cls = make_polars_extension_type(name, pa.binary(16), None) - inst = cls() - assert inst.ext_name() == name - - -def test_make_polars_extension_type_with_metadata(): - """make_polars_extension_type captures metadata in the class.""" - name = _unique_name() - cls = make_polars_extension_type(name, pa.large_utf8(), "test.metadata") - # Instantiating should not raise; ext_name is correct. - inst = cls() - assert inst.ext_name() == name -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -cd /home/kurouto/kurouto-jobs/fccdf92d-a25e-4477-ae00-a1ee2b6dc236/orcapod-python -uv run pytest tests/test_extension_types/test_registry.py::test_make_polars_extension_type_returns_class -v -``` - -Expected: `ImportError` — `make_polars_extension_type` does not exist yet. - -- [ ] **Step 3: Implement `make_polars_extension_type` in `registry.py`** - -Add after `make_arrow_extension_type` (around line 98), before the `LogicalTypeRegistry` class: - -```python -def make_polars_extension_type( - extension_name: str, - arrow_storage_type: pa.DataType, - metadata: str | None = None, -) -> type[pl.BaseExtension]: - """Synthesise and return a ``pl.BaseExtension`` subclass. - - Derives the Polars storage dtype from *arrow_storage_type* via - ``pl.from_arrow``. Returns the *class*; callers instantiate it inside - ``get_polars_extension_type()``. - - The returned class uses the Arrow extension name as its registration name - (the same name passed to ``pl.register_extension_type``), so that Polars - correctly maps Arrow extension columns on read. - - Args: - extension_name: The extension type name used for Polars registration. - Must match the Arrow extension name so Polars can round-trip the - type through Arrow IPC. - arrow_storage_type: The Arrow storage type. Converted once to the - corresponding Polars dtype via ``pl.from_arrow``. - metadata: Optional metadata string stored as ``metadata_str`` in the - Polars extension. Defaults to ``None``. - - Returns: - A ``pl.BaseExtension`` subclass. Call it with no arguments to obtain - an instance suitable for passing to ``pl.register_extension_type`` or - returning from ``get_polars_extension_type()``. - """ - _name = extension_name - _polars_dtype = pl.from_arrow(pa.array([], type=arrow_storage_type)).dtype - _metadata = metadata - - def __init__(self: pl.BaseExtension) -> None: - pl.BaseExtension.__init__(self, _name, _polars_dtype, _metadata) - - @classmethod # type: ignore[misc] - def ext_from_params( - cls: type[pl.BaseExtension], - ext_name: str, - storage_dtype: pl.PolarsDataType, - metadata_str: str | None, - ) -> pl.BaseExtension: - return cls() - - return type( - f"_PolarsExt_{_sanitize(extension_name)}", - (pl.BaseExtension,), - { - "__init__": __init__, - "ext_from_params": ext_from_params, - }, - ) -``` - -- [ ] **Step 4: Export `make_polars_extension_type` from `extension_types/__init__.py`** - -In `src/orcapod/extension_types/__init__.py`, update the import line and `__all__`: - -```python -from .registry import LogicalTypeRegistry, make_arrow_extension_type, make_polars_extension_type -``` - -And add `"make_polars_extension_type"` to `__all__`: - -```python -__all__ = [ - "LogicalType", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "make_polars_extension_type", - "default_logical_type_registry", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", -] -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -k "polars_extension_type" -v -``` - -Expected: All 5 new tests PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/registry.py \ - src/orcapod/extension_types/__init__.py \ - tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add make_polars_extension_type helper" -``` - ---- - -### Task 2: `LogicalTypeRegistry` `logical_types` constructor param - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Write the failing tests** - -Add after the existing `test_get_by_arrow_extension_name_miss` test, before the PyArrow global registry tests section: - -```python -# --------------------------------------------------------------------------- -# LogicalTypeRegistry constructor logical_types param tests -# --------------------------------------------------------------------------- - -def test_registry_init_with_logical_types_preregisters(): - """LogicalTypeRegistry(logical_types=[lt]) makes the type immediately retrievable.""" - lt = _make_stub() - registry = LogicalTypeRegistry(logical_types=[lt]) - assert registry.get_by_logical_name(lt.logical_type_name) is lt - assert registry.get_by_python_type(lt.python_type) is lt - assert registry.get_by_arrow_extension_name(lt.get_arrow_extension_type().extension_name) is lt - - -def test_registry_init_with_none_is_empty(): - """LogicalTypeRegistry(logical_types=None) starts empty without error.""" - registry = LogicalTypeRegistry(logical_types=None) - assert registry.get_by_logical_name("anything") is None - - -def test_registry_init_with_empty_list_is_empty(): - """LogicalTypeRegistry(logical_types=[]) starts empty without error.""" - registry = LogicalTypeRegistry(logical_types=[]) - assert registry.get_by_logical_name("anything") is None - - -def test_registry_init_with_multiple_logical_types(): - """LogicalTypeRegistry(logical_types=[lt1, lt2]) registers both.""" - lt1 = _make_stub(py_type=int) - lt2 = _make_stub(py_type=float) - registry = LogicalTypeRegistry(logical_types=[lt1, lt2]) - assert registry.get_by_logical_name(lt1.logical_type_name) is lt1 - assert registry.get_by_logical_name(lt2.logical_type_name) is lt2 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_registry.py::test_registry_init_with_logical_types_preregisters -v -``` - -Expected: FAIL — `LogicalTypeRegistry.__init__` does not accept `logical_types` argument. - -- [ ] **Step 3: Update `LogicalTypeRegistry.__init__` in `registry.py`** - -Replace the current `__init__` method (lines 121–124): - -```python -# OLD -def __init__(self) -> None: - self._by_logical_name: dict[str, LogicalType] = {} - self._by_arrow_name: dict[str, LogicalType] = {} - self._by_python_type: dict[type, LogicalType] = {} -``` - -With: - -```python -def __init__(self, logical_types: list[LogicalType] | None = None) -> None: - self._by_logical_name: dict[str, LogicalType] = {} - self._by_arrow_name: dict[str, LogicalType] = {} - self._by_python_type: dict[type, LogicalType] = {} - for lt in (logical_types or []): - self.register(lt) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -k "registry_init" -v -``` - -Expected: All 4 new tests PASS. Also run the full registry suite to confirm no regressions: - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: All tests PASS (the last 6 `default_logical_type_registry` tests still reference the old module-level instance and will continue passing for now — they are removed in Task 6). - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/registry.py \ - tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add logical_types constructor param to LogicalTypeRegistry" -``` - ---- - -### Task 3: `LogicalPath` and `LogicalUPath` implementations - -**Files:** -- Create: `src/orcapod/extension_types/builtin_logical_types.py` -- Create: `tests/test_extension_types/test_builtin_logical_types.py` - -- [ ] **Step 1: Create the test file with failing tests for `LogicalPath` and `LogicalUPath`** - -Create `tests/test_extension_types/test_builtin_logical_types.py`: - -```python -"""Tests for built-in LogicalType implementations (LogicalPath, LogicalUPath, LogicalUUID).""" - -from __future__ import annotations - -import pathlib -import uuid as uuid_module -import warnings - -import polars as pl -import pyarrow as pa -import pytest -from upath import UPath - -from orcapod.extension_types.protocols import LogicalType -from orcapod.extension_types.registry import LogicalTypeRegistry - - -# --------------------------------------------------------------------------- -# LogicalPath tests -# --------------------------------------------------------------------------- - - -def test_logical_path_isinstance_logical_type(): - """LogicalPath() satisfies the LogicalType runtime-checkable protocol.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - - assert isinstance(LogicalPath(), LogicalType) - - -def test_logical_path_logical_type_name(): - from orcapod.extension_types.builtin_logical_types import LogicalPath - - assert LogicalPath().logical_type_name == "pathlib.Path" - - -def test_logical_path_python_type(): - from orcapod.extension_types.builtin_logical_types import LogicalPath - - assert LogicalPath().python_type is pathlib.Path - - -def test_logical_path_arrow_ext_name(): - """get_arrow_extension_type().extension_name is 'pathlib.Path'.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - - assert LogicalPath().get_arrow_extension_type().extension_name == "pathlib.Path" - - -def test_logical_path_arrow_ext_storage_type(): - """Arrow extension storage type is pa.large_string().""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - - assert LogicalPath().get_arrow_extension_type().storage_type == pa.large_string() - - -def test_logical_path_get_arrow_extension_type_is_cached(): - """get_arrow_extension_type() returns the same object on repeated calls.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - - lt = LogicalPath() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - -def test_logical_path_get_polars_extension_type_is_cached(): - """get_polars_extension_type() returns the same object on repeated calls.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - - lt = LogicalPath() - assert lt.get_polars_extension_type() is lt.get_polars_extension_type() - - -def test_logical_path_round_trip(): - """Path -> python_to_storage -> storage_to_python -> Path is identity.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - - lt = LogicalPath() - p = pathlib.Path("/tmp/foo/bar.txt") - assert lt.storage_to_python(lt.python_to_storage(p)) == p - - -def test_logical_path_python_to_storage_returns_string(): - from orcapod.extension_types.builtin_logical_types import LogicalPath - - lt = LogicalPath() - result = lt.python_to_storage(pathlib.Path("/tmp/test")) - assert isinstance(result, str) - assert result == "/tmp/test" - - -# --------------------------------------------------------------------------- -# LogicalUPath tests -# --------------------------------------------------------------------------- - - -def test_logical_upath_isinstance_logical_type(): - """LogicalUPath() satisfies the LogicalType runtime-checkable protocol.""" - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - assert isinstance(LogicalUPath(), LogicalType) - - -def test_logical_upath_logical_type_name(): - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - assert LogicalUPath().logical_type_name == "upath.UPath" - - -def test_logical_upath_python_type(): - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - assert LogicalUPath().python_type is UPath - - -def test_logical_upath_arrow_ext_name(): - """get_arrow_extension_type().extension_name is 'upath.UPath'.""" - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - assert LogicalUPath().get_arrow_extension_type().extension_name == "upath.UPath" - - -def test_logical_upath_arrow_ext_storage_type(): - """Arrow extension storage type is pa.large_string().""" - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - assert LogicalUPath().get_arrow_extension_type().storage_type == pa.large_string() - - -def test_logical_upath_get_arrow_extension_type_is_cached(): - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - lt = LogicalUPath() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - -def test_logical_upath_round_trip(): - """UPath -> python_to_storage -> storage_to_python -> UPath is identity.""" - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - lt = LogicalUPath() - up = UPath("s3://bucket/key/file.txt") - assert lt.storage_to_python(lt.python_to_storage(up)) == up - - -def test_logical_upath_python_to_storage_returns_string(): - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - lt = LogicalUPath() - result = lt.python_to_storage(UPath("s3://bucket/key")) - assert isinstance(result, str) - assert result == "s3://bucket/key" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py::test_logical_path_isinstance_logical_type -v -``` - -Expected: `ModuleNotFoundError` — `builtin_logical_types` does not exist yet. - -- [ ] **Step 3: Create `src/orcapod/extension_types/builtin_logical_types.py` with `LogicalPath` and `LogicalUPath`** - -```python -"""Built-in LogicalType implementations for orcapod. - -Provides three built-in logical types registered into the default -``DataContext.logical_type_registry`` via ``contexts/data/v0.1.json``: - -- ``LogicalPath``: maps ``pathlib.Path`` ↔ Arrow large_string extension "pathlib.Path" -- ``LogicalUPath``: maps ``upath.UPath`` ↔ Arrow large_string extension "upath.UPath" -- ``LogicalUUID``: maps ``uuid.UUID`` ↔ PyArrow built-in ``pa.uuid()`` ("arrow.uuid") - -Note: - All imports from orcapod.extension_types use direct submodule paths - (e.g. ``from orcapod.extension_types.registry import ...``) rather than - the package ``__init__`` to avoid circular imports when the context system - loads this module at startup. -""" - -from __future__ import annotations - -import pathlib -import uuid as _uuid_module -from typing import TYPE_CHECKING, Any - -from upath import UPath - -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - - -class LogicalPath: - """Logical type for ``pathlib.Path``. - - Stores paths as Arrow large strings using the custom extension type - ``"pathlib.Path"`` with metadata ``b"orcapod.builtin"``. - - Example: - >>> lt = LogicalPath() - >>> lt.python_to_storage(pathlib.Path("/tmp/foo")) - '/tmp/foo' - >>> lt.storage_to_python('/tmp/foo') - PosixPath('/tmp/foo') - """ - - _arrow_ext_class = make_arrow_extension_type( - "pathlib.Path", pa.large_string(), b"orcapod.builtin" - ) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type( - "pathlib.Path", pa.large_string(), "orcapod.builtin" - ) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "pathlib.Path" - python_type: type = pathlib.Path - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for ``pathlib.Path``. - - Returns: - A cached ``pa.ExtensionType`` instance with extension name - ``"pathlib.Path"`` and storage type ``pa.large_string()``. - """ - if LogicalPath._arrow_ext is None: - LogicalPath._arrow_ext = LogicalPath._arrow_ext_class() - return LogicalPath._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``pathlib.Path``. - - Returns: - A cached ``pl.BaseExtension`` instance registered under - ``"pathlib.Path"``. - """ - if LogicalPath._polars_ext is None: - LogicalPath._polars_ext = LogicalPath._polars_ext_class() - return LogicalPath._polars_ext - - def python_to_storage(self, value: Any) -> str: - """Convert a ``pathlib.Path`` to its string representation. - - Args: - value: A ``pathlib.Path`` instance. - - Returns: - The string form of the path (e.g. ``"/tmp/foo"``). - """ - return str(value) - - def storage_to_python(self, storage_value: Any) -> pathlib.Path: - """Reconstruct a ``pathlib.Path`` from its string representation. - - Args: - storage_value: A string path as stored in Arrow. - - Returns: - A ``pathlib.Path`` instance. - """ - return pathlib.Path(storage_value) - - -class LogicalUPath: - """Logical type for ``upath.UPath``. - - Stores paths as Arrow large strings using the custom extension type - ``"upath.UPath"`` with metadata ``b"orcapod.builtin"``. - - Example: - >>> lt = LogicalUPath() - >>> lt.python_to_storage(UPath("s3://bucket/key")) - 's3://bucket/key' - >>> lt.storage_to_python("s3://bucket/key") - UPath('s3://bucket/key') - """ - - _arrow_ext_class = make_arrow_extension_type( - "upath.UPath", pa.large_string(), b"orcapod.builtin" - ) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type( - "upath.UPath", pa.large_string(), "orcapod.builtin" - ) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "upath.UPath" - python_type: type = UPath - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for ``upath.UPath``. - - Returns: - A cached ``pa.ExtensionType`` instance with extension name - ``"upath.UPath"`` and storage type ``pa.large_string()``. - """ - if LogicalUPath._arrow_ext is None: - LogicalUPath._arrow_ext = LogicalUPath._arrow_ext_class() - return LogicalUPath._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``upath.UPath``. - - Returns: - A cached ``pl.BaseExtension`` instance registered under - ``"upath.UPath"``. - """ - if LogicalUPath._polars_ext is None: - LogicalUPath._polars_ext = LogicalUPath._polars_ext_class() - return LogicalUPath._polars_ext - - def python_to_storage(self, value: Any) -> str: - """Convert a ``upath.UPath`` to its string representation. - - Args: - value: A ``upath.UPath`` instance. - - Returns: - The string form of the path (e.g. ``"s3://bucket/key"``). - """ - return str(value) - - def storage_to_python(self, storage_value: Any) -> UPath: - """Reconstruct a ``upath.UPath`` from its string representation. - - Args: - storage_value: A string path as stored in Arrow. - - Returns: - A ``upath.UPath`` instance. - """ - return UPath(storage_value) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -k "logical_path or logical_upath" -v -``` - -Expected: All `LogicalPath` and `LogicalUPath` tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/builtin_logical_types.py \ - tests/test_extension_types/test_builtin_logical_types.py -git commit -m "feat(extension_types): implement LogicalPath and LogicalUPath" -``` - ---- - -### Task 4: `LogicalUUID` implementation - -**Files:** -- Modify: `src/orcapod/extension_types/builtin_logical_types.py` -- Modify: `tests/test_extension_types/test_builtin_logical_types.py` - -- [ ] **Step 1: Write the failing tests for `LogicalUUID`** - -Append to `tests/test_extension_types/test_builtin_logical_types.py`: - -```python -# --------------------------------------------------------------------------- -# LogicalUUID tests -# --------------------------------------------------------------------------- - - -def test_logical_uuid_isinstance_logical_type(): - """LogicalUUID() satisfies the LogicalType runtime-checkable protocol.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - assert isinstance(LogicalUUID(), LogicalType) - - -def test_logical_uuid_logical_type_name(): - """logical_type_name is 'uuid.UUID', not the Arrow extension name.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - assert LogicalUUID().logical_type_name == "uuid.UUID" - - -def test_logical_uuid_python_type(): - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - assert LogicalUUID().python_type is uuid_module.UUID - - -def test_logical_uuid_arrow_ext_name_is_arrow_uuid(): - """Arrow extension name is 'arrow.uuid', intentionally different from logical_type_name.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - assert lt.get_arrow_extension_type().extension_name == "arrow.uuid" - assert lt.logical_type_name != lt.get_arrow_extension_type().extension_name - - -def test_logical_uuid_get_arrow_extension_type_returns_pa_uuid(): - """get_arrow_extension_type() returns PyArrow's built-in pa.uuid() type.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - assert lt.get_arrow_extension_type() == pa.uuid() - - -def test_logical_uuid_get_arrow_extension_type_is_cached(): - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - -def test_logical_uuid_get_polars_extension_type_is_cached(): - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - assert lt.get_polars_extension_type() is lt.get_polars_extension_type() - - -def test_logical_uuid_round_trip(): - """UUID -> python_to_storage -> storage_to_python -> UUID is identity.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - u = uuid_module.uuid4() - assert lt.storage_to_python(lt.python_to_storage(u)) == u - - -def test_logical_uuid_python_to_storage_returns_bytes(): - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - u = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - result = lt.python_to_storage(u) - assert isinstance(result, bytes) - assert len(result) == 16 - - -def test_logical_uuid_storage_to_python_accepts_bytes(): - """storage_to_python works when storage_value is plain bytes.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - lt = LogicalUUID() - u = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - recovered = lt.storage_to_python(u.bytes) - assert recovered == u - - -def test_logical_uuid_registration_does_not_raise(): - """Registering LogicalUUID succeeds even though pa.uuid() is already in PyArrow's registry.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - registry = LogicalTypeRegistry() - lt = LogicalUUID() - registry.register(lt) # should NOT raise - assert registry.get_by_logical_name("uuid.UUID") is lt - assert registry.get_by_arrow_extension_name("arrow.uuid") is lt -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py::test_logical_uuid_isinstance_logical_type -v -``` - -Expected: `ImportError` — `LogicalUUID` does not exist yet. - -- [ ] **Step 3: Add `LogicalUUID` to `builtin_logical_types.py`** - -Append to the end of `src/orcapod/extension_types/builtin_logical_types.py`: - -```python -class LogicalUUID: - """Logical type for ``uuid.UUID``. - - Uses PyArrow's built-in ``pa.uuid()`` extension type (``"arrow.uuid"``) - which stores UUID values as 16-byte binary (``pa.binary(16)``). - - Note: - ``logical_type_name`` (``"uuid.UUID"``) intentionally differs from - the Arrow extension name (``"arrow.uuid"``). The - ``LogicalTypeRegistry`` stores both bindings so that lookups by - either key resolve to this same instance. - - Example: - >>> import uuid - >>> lt = LogicalUUID() - >>> u = uuid.uuid4() - >>> lt.storage_to_python(lt.python_to_storage(u)) == u - True - """ - - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("arrow.uuid", pa.binary(16), None) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "uuid.UUID" - python_type: type = _uuid_module.UUID - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return PyArrow's built-in ``pa.uuid()`` extension type. - - Returns: - A cached ``pa.uuid()`` instance (Arrow extension name ``"arrow.uuid"``, - storage type ``pa.binary(16)``). - """ - if LogicalUUID._arrow_ext is None: - LogicalUUID._arrow_ext = pa.uuid() - return LogicalUUID._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``arrow.uuid``. - - Returns: - A cached ``pl.BaseExtension`` instance registered under - ``"arrow.uuid"`` (matches the Arrow extension name, not the - logical type name). - """ - if LogicalUUID._polars_ext is None: - LogicalUUID._polars_ext = LogicalUUID._polars_ext_class() - return LogicalUUID._polars_ext - - def python_to_storage(self, value: Any) -> bytes: - """Convert a ``uuid.UUID`` to its 16-byte binary representation. - - Args: - value: A ``uuid.UUID`` instance. - - Returns: - A 16-byte ``bytes`` object (big-endian byte order, as per - ``uuid.UUID.bytes``). - """ - return value.bytes - - def storage_to_python(self, storage_value: Any) -> _uuid_module.UUID: - """Reconstruct a ``uuid.UUID`` from its 16-byte binary representation. - - Args: - storage_value: A bytes-like object of length 16. - - Returns: - A ``uuid.UUID`` instance. - """ - return _uuid_module.UUID(bytes=bytes(storage_value)) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v -``` - -Expected: All tests in the file PASS (LogicalPath, LogicalUPath, and LogicalUUID). - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/builtin_logical_types.py \ - tests/test_extension_types/test_builtin_logical_types.py -git commit -m "feat(extension_types): implement LogicalUUID" -``` - ---- - -### Task 5: Wire built-in types into `DataContext` - -**Files:** -- Modify: `src/orcapod/contexts/core.py` -- Modify: `src/orcapod/contexts/registry.py` -- Modify: `src/orcapod/contexts/data/v0.1.json` -- Modify: `src/orcapod/contexts/data/schemas/context_schema.json` -- Modify: `src/orcapod/contexts/__init__.py` -- Modify: `tests/test_extension_types/test_builtin_logical_types.py` - -This task wires everything together. The integration tests are written first, but they cannot pass until the DataContext and JSON spec are updated. Do all the sub-steps in a single commit. - -- [ ] **Step 1: Write the failing integration tests** - -Append to `tests/test_extension_types/test_builtin_logical_types.py`: - -```python -# --------------------------------------------------------------------------- -# Default context integration tests -# --------------------------------------------------------------------------- - - -def test_default_context_has_logical_type_registry(): - """DataContext has a logical_type_registry attribute.""" - from orcapod.contexts import get_default_context - - ctx = get_default_context() - assert hasattr(ctx, "logical_type_registry") - - -def test_default_context_registry_has_logical_path(): - """Default registry returns LogicalPath for 'pathlib.Path'.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalPath - - registry = get_default_context().logical_type_registry - lt = registry.get_by_logical_name("pathlib.Path") - assert isinstance(lt, LogicalPath) - - -def test_default_context_registry_lookup_by_python_type_path(): - """Default registry routes pathlib.Path to LogicalPath.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalPath - - registry = get_default_context().logical_type_registry - lt = registry.get_by_python_type(pathlib.Path) - assert isinstance(lt, LogicalPath) - - -def test_default_context_registry_lookup_by_arrow_name_path(): - """Default registry routes 'pathlib.Path' arrow ext name to LogicalPath.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalPath - - registry = get_default_context().logical_type_registry - lt = registry.get_by_arrow_extension_name("pathlib.Path") - assert isinstance(lt, LogicalPath) - - -def test_default_context_registry_has_logical_upath(): - """Default registry returns LogicalUPath for 'upath.UPath'.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - registry = get_default_context().logical_type_registry - lt = registry.get_by_logical_name("upath.UPath") - assert isinstance(lt, LogicalUPath) - - -def test_default_context_registry_lookup_by_python_type_upath(): - """Default registry routes UPath to LogicalUPath.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalUPath - - registry = get_default_context().logical_type_registry - lt = registry.get_by_python_type(UPath) - assert isinstance(lt, LogicalUPath) - - -def test_default_context_registry_has_logical_uuid(): - """Default registry returns LogicalUUID for 'uuid.UUID'.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - registry = get_default_context().logical_type_registry - lt = registry.get_by_logical_name("uuid.UUID") - assert isinstance(lt, LogicalUUID) - - -def test_default_context_registry_lookup_by_arrow_name_uuid(): - """Default registry routes 'arrow.uuid' arrow ext name to LogicalUUID.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.builtin_logical_types import LogicalUUID - - registry = get_default_context().logical_type_registry - lt = registry.get_by_arrow_extension_name("arrow.uuid") - assert isinstance(lt, LogicalUUID) - - -def test_default_context_registry_uuid_logical_name_differs_from_arrow_name(): - """The same LogicalUUID instance is found by both 'uuid.UUID' and 'arrow.uuid'.""" - from orcapod.contexts import get_default_context - - registry = get_default_context().logical_type_registry - by_logical = registry.get_by_logical_name("uuid.UUID") - by_arrow = registry.get_by_arrow_extension_name("arrow.uuid") - assert by_logical is by_arrow - - -def test_get_default_logical_type_registry_returns_same_as_context(): - """get_default_logical_type_registry() is the same object as get_default_context().logical_type_registry.""" - from orcapod.contexts import get_default_context, get_default_logical_type_registry - - assert get_default_logical_type_registry() is get_default_context().logical_type_registry - - -def test_default_context_idempotent_registry(): - """Calling get_default_context() twice returns the same LogicalTypeRegistry instance.""" - from orcapod.contexts import get_default_context - - r1 = get_default_context().logical_type_registry - r2 = get_default_context().logical_type_registry - assert r1 is r2 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py::test_default_context_has_logical_type_registry -v -``` - -Expected: FAIL — `DataContext` has no `logical_type_registry` attribute. - -- [ ] **Step 3: Add `logical_type_registry` field to `DataContext` in `core.py`** - -Current `core.py` imports (lines 1–16): - -```python -""" -Core data structures and exceptions for the OrcaPod context system. -... -""" - -from dataclasses import dataclass - -from orcapod.hashing.semantic_hashing.type_handler_registry import TypeHandlerRegistry -from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - SemanticHasherProtocol, -) -from orcapod.protocols.semantic_types_protocols import TypeConverterProtocol -``` - -Add one import and one field. The final `core.py` content: - -```python -""" -Core data structures and exceptions for the OrcaPod context system. - -This module defines the basic types and exceptions used throughout -the context management system. -""" - -from dataclasses import dataclass - -from orcapod.extension_types.registry import LogicalTypeRegistry -from orcapod.hashing.semantic_hashing.type_handler_registry import TypeHandlerRegistry -from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - SemanticHasherProtocol, -) -from orcapod.protocols.semantic_types_protocols import TypeConverterProtocol - - -@dataclass -class DataContext: - """ - Data context containing all versioned components needed for data interpretation. - - A DataContext represents a specific version of the OrcaPod system configuration, - including semantic type registries, hashers, and other components that affect - how data is processed and interpreted. - - Attributes: - context_key: Unique identifier (e.g., "std:v0.1:default") - version: Version string (e.g., "v0.1") - description: Human-readable description of this context - semantic_type_registry: Registry of semantic type converters - arrow_hasher: Arrow table hasher for this context - semantic_hasher: General semantic hasher for this context - type_handler_registry: Registry of TypeHandlerProtocol instances for SemanticHasherProtocol - logical_type_registry: Registry of LogicalType instances (Path, UPath, UUID, etc.) - """ - - context_key: str - version: str - description: str - type_converter: TypeConverterProtocol - arrow_hasher: ArrowHasherProtocol - semantic_hasher: SemanticHasherProtocol # this is the currently the JSON hasher - type_handler_registry: TypeHandlerRegistry - logical_type_registry: LogicalTypeRegistry - - -class ContextValidationError(Exception): - """Raised when context validation fails.""" - - pass - - -class ContextResolutionError(Exception): - """Raised when context cannot be resolved.""" - - pass -``` - -- [ ] **Step 4: Update `contexts/registry.py` — add `logical_type_registry` to required fields and `_create_context_from_spec`** - -In `_load_spec_file` (around line 148), add `"logical_type_registry"` to `required_fields`: - -```python -required_fields = [ - "context_key", - "version", - "type_converter", - "arrow_hasher", - "semantic_hasher", - "type_handler_registry", - "logical_type_registry", -] -``` - -In `_create_context_from_spec` (around line 296), add `logical_type_registry` to the `DataContext(...)` call: - -```python -return DataContext( - context_key=context_key, - version=version, - description=description, - type_converter=ref_lut["type_converter"], - arrow_hasher=ref_lut["arrow_hasher"], - semantic_hasher=ref_lut["semantic_hasher"], - type_handler_registry=ref_lut["type_handler_registry"], - logical_type_registry=ref_lut["logical_type_registry"], -) -``` - -- [ ] **Step 5: Add `logical_type_registry` entry to `v0.1.json`** - -In `src/orcapod/contexts/data/v0.1.json`, add the following JSON block before the `"metadata"` key (after the `"semantic_hasher"` block): - -```json - "logical_type_registry": { - "_class": "orcapod.extension_types.registry.LogicalTypeRegistry", - "_config": { - "logical_types": [ - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUUID", - "_config": {} - } - ] - } - }, -``` - -The full updated `v0.1.json` after the edit: - -```json -{ - "context_key": "std:v0.1:default", - "version": "v0.1", - "description": "Initial stable release with basic Path semantic type support", - "file_hasher": { - "_class": "orcapod.hashing.file_hashers.BasicFileHasher", - "_config": { - "algorithm": "sha256" - } - }, - "semantic_registry": { - "_class": "orcapod.semantic_types.semantic_registry.SemanticTypeRegistry", - "_config": { - "converters": { - "upath": { - "_class": "orcapod.semantic_types.semantic_struct_converters.UPathStructConverter", - "_config": { - "file_hasher": {"_ref": "file_hasher"} - } - }, - "path": { - "_class": "orcapod.semantic_types.semantic_struct_converters.PythonPathStructConverter", - "_config": { - "file_hasher": {"_ref": "file_hasher"} - } - } - } - } - }, - "arrow_hasher": { - "_class": "orcapod.hashing.arrow_hashers.StarfixArrowHasher", - "_config": { - "hasher_id": "arrow_v0.1", - "semantic_registry": { - "_ref": "semantic_registry" - } - } - }, - "type_converter": { - "_class": "orcapod.semantic_types.universal_converter.UniversalTypeConverter", - "_config": { - "semantic_registry": { - "_ref": "semantic_registry" - } - } - }, - "function_info_extractor": { - "_class": "orcapod.hashing.semantic_hashing.function_info_extractors.FunctionSignatureExtractor", - "_config": { - "include_module": true, - "include_defaults": true - } - }, - "type_handler_registry": { - "_class": "orcapod.hashing.semantic_hashing.type_handler_registry.TypeHandlerRegistry", - "_config": { - "handlers": [ - [{"_type": "builtins.bytes"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.BytesHandler", "_config": {}}], - [{"_type": "builtins.bytearray"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.BytesHandler", "_config": {}}], - [{"_type": "pathlib.Path"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PathContentHandler", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], - [{"_type": "upath.core.UPath"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UPathContentHandler", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], - [{"_type": "uuid.UUID"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UUIDHandler", "_config": {}}], - [{"_type": "types.FunctionType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FunctionHandler", "_config": {"function_info_extractor": {"_ref": "function_info_extractor"}}}], - [{"_type": "types.BuiltinFunctionType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FunctionHandler", "_config": {"function_info_extractor": {"_ref": "function_info_extractor"}}}], - [{"_type": "types.MethodType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FunctionHandler", "_config": {"function_info_extractor": {"_ref": "function_info_extractor"}}}], - [{"_type": "builtins.type"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.TypeObjectHandler", "_config": {}}], - [{"_type": "types.GenericAlias"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.GenericAliasHandler", "_config": {}}], - [{"_type": "types.UnionType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UnionTypeHandler", "_config": {}}], - [{"_type": "typing._GenericAlias"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.GenericAliasHandler", "_config": {}}], - [{"_type": "typing._SpecialForm"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.SpecialFormHandler", "_config": {}}], - [{"_type": "pyarrow.Table"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.ArrowTableHandler", "_config": {"arrow_hasher": {"_ref": "arrow_hasher"}}}], - [{"_type": "pyarrow.RecordBatch"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.ArrowTableHandler", "_config": {"arrow_hasher": {"_ref": "arrow_hasher"}}}] - ] - } - }, - "semantic_hasher": { - "_class": "orcapod.hashing.semantic_hashing.semantic_hasher.BaseSemanticHasher", - "_config": { - "hasher_id": "semantic_v0.1", - "type_handler_registry": { - "_ref": "type_handler_registry" - } - } - }, - "logical_type_registry": { - "_class": "orcapod.extension_types.registry.LogicalTypeRegistry", - "_config": { - "logical_types": [ - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUUID", - "_config": {} - } - ] - } - }, - "metadata": { - "created_date": "2025-08-01", - "author": "OrcaPod Core Team", - "changelog": [ - "Initial release with Path semantic type support", - "Basic SHA-256 hashing for files and objects", - "Arrow logical serialization method", - "Introduced arrow_v0.1 StarfixArrowHasher using starfix ArrowDigester for cross-language-compatible Arrow hashing" - ] - } -} -``` - -- [ ] **Step 6: Add `logical_type_registry` to `context_schema.json`** - -In `src/orcapod/contexts/data/schemas/context_schema.json`: - -Add `"logical_type_registry"` to the `"required"` array (after `"type_handler_registry"`): - -```json -"required": [ - "context_key", - "version", - "semantic_registry", - "type_converter", - "arrow_hasher", - "semantic_hasher", - "type_handler_registry", - "logical_type_registry" -], -``` - -Add `"logical_type_registry"` entry to the `"properties"` object (after `"type_handler_registry"`): - -```json -"logical_type_registry": { - "$ref": "#/$defs/objectspec", - "description": "ObjectSpec for the LogicalTypeRegistry (Path, UPath, UUID built-ins)" -}, -``` - -- [ ] **Step 7: Add `get_default_logical_type_registry()` to `contexts/__init__.py`** - -In `src/orcapod/contexts/__init__.py`, add after `get_default_type_converter()`: - -```python -def get_default_logical_type_registry() -> "LogicalTypeRegistry": - """Get the default logical type registry. - - Returns: - ``LogicalTypeRegistry`` instance from the default context. - """ - return get_default_context().logical_type_registry -``` - -Add the import at the top of the file (after the `from orcapod.protocols` imports): - -```python -from orcapod.extension_types.registry import LogicalTypeRegistry -``` - -Add `"get_default_logical_type_registry"` to `__all__`. - -The updated `__all__` in `contexts/__init__.py`: - -```python -__all__ = [ - # Core types - "DataContext", - "ContextValidationError", - "ContextResolutionError", - # Main functions - "resolve_context", - "get_available_contexts", - "get_context_info", - "get_default_context", - # Convenience accessors - "get_default_semantic_hasher", - "get_default_arrow_hasher", - "get_default_type_converter", - "get_default_logical_type_registry", - # Management functions - "set_default_context_version", - "validate_all_contexts", - "reload_contexts", - # Advanced usage - "create_registry", - "JSONDataContextRegistry", -] -``` - -- [ ] **Step 8: Run the integration tests** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v -``` - -Expected: All tests PASS, including the new integration tests. - -- [ ] **Step 9: Run the full test suite to check for regressions** - -```bash -uv run pytest tests/ -v --tb=short -``` - -Expected: All previously-passing tests still PASS. The 6 `default_logical_type_registry` tests in `test_registry.py` still pass (the module-level variable is still there; we remove it next). - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/contexts/core.py \ - src/orcapod/contexts/registry.py \ - src/orcapod/contexts/data/v0.1.json \ - src/orcapod/contexts/data/schemas/context_schema.json \ - src/orcapod/contexts/__init__.py \ - tests/test_extension_types/test_builtin_logical_types.py -git commit -m "feat(contexts): add logical_type_registry to DataContext and v0.1 context" -``` - ---- - -### Task 6: Remove `default_logical_type_registry` and clean up stale tests - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `tests/test_extension_types/test_registry.py` - -The module-level `default_logical_type_registry` in `extension_types/__init__.py` is replaced by the context-scoped registry. This task removes it and deletes the 6 tests that relied on it. - -- [ ] **Step 1: Remove `default_logical_type_registry` from `extension_types/__init__.py`** - -Replace the current content of `src/orcapod/extension_types/__init__.py`: - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for logical types that map -between Python objects and their Arrow/Polars extension type representation. - -Built-in registrations (``LogicalPath``, ``LogicalUPath``, ``LogicalUUID``) are -wired into ``DataContext`` via ``contexts/data/v0.1.json``. The primary access -path for the default registry is: - -- ``get_default_context().logical_type_registry`` -- ``get_default_logical_type_registry()`` (from ``orcapod.contexts``) -""" - -from __future__ import annotations - -from .protocols import LogicalType -from .registry import LogicalTypeRegistry, make_arrow_extension_type, make_polars_extension_type -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema - -__all__ = [ - "LogicalType", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "make_polars_extension_type", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", -] -``` - -- [ ] **Step 2: Remove the 6 stale `default_logical_type_registry` tests from `test_registry.py`** - -Delete the entire section at the end of `tests/test_extension_types/test_registry.py` (lines 450–532): - -```python -# --------------------------------------------------------------------------- -# default_logical_type_registry tests -# --------------------------------------------------------------------------- - -def test_logical_type_registry_module_instance(): - ... - -def test_default_registry_is_same_object_across_imports(): - ... - -def test_default_registry_register_and_lookup(): - ... - -def test_default_registry_register_idempotent(): - ... - -def test_default_registry_populates_arrow_global(): - ... - -def test_default_registry_populates_polars_global(): - ... -``` - -These tests are superseded by the integration tests in `test_builtin_logical_types.py`. - -- [ ] **Step 3: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short -``` - -Expected: All tests PASS. The 6 removed tests no longer exist. No regressions. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py \ - tests/test_extension_types/test_registry.py -git commit -m "refactor(extension_types): remove default_logical_type_registry module-level variable" -``` - ---- - -## Self-Review - -### Spec coverage check - -| Spec requirement | Covered by | -|---|---| -| `LogicalPath` implementation | Task 3 | -| `LogicalUPath` implementation | Task 3 | -| `LogicalUUID` implementation (with `pa.uuid()`) | Task 4 | -| `make_polars_extension_type` helper | Task 1 | -| `LogicalTypeRegistry.__init__` `logical_types` param | Task 2 | -| `DataContext.logical_type_registry` field | Task 5, Step 3 | -| `v0.1.json` `logical_type_registry` entry | Task 5, Step 5 | -| `context_schema.json` update | Task 5, Step 6 | -| `get_default_logical_type_registry()` convenience function | Task 5, Step 7 | -| Remove `default_logical_type_registry` from `__init__.py` | Task 6, Step 1 | -| Protocol conformance tests | Task 3 & 4 | -| Property value tests | Task 3 & 4 | -| Conversion round-trip tests | Task 3 & 4 | -| Default context registration tests | Task 5, Step 1 | -| Pre-existing Arrow type tolerance test (`LogicalUUID`) | Task 4, Step 1 | -| Idempotence test (context caching) | Task 5, Step 1 | -| UUID `logical_type_name` ≠ Arrow ext name test | Task 4, Step 1 | -| Circular import avoidance (submodule imports) | Task 3, Step 3 (in `builtin_logical_types.py`) | -| Class-level caching for extension type instances | Task 3, Step 3 & Task 4, Step 3 | -| Export `make_polars_extension_type` from `__init__.py` | Task 1, Step 4 | - -### Type consistency check - -- `make_polars_extension_type(name, arrow_storage_type, metadata)` — used consistently in Task 1 (definition) and Task 3/4 (class-body calls). -- `LogicalTypeRegistry(logical_types=[...])` — defined in Task 2, used in Task 5 JSON spec. -- `DataContext.logical_type_registry` field — added in Task 5 Step 3, passed in `_create_context_from_spec` in Task 5 Step 4. -- `get_default_logical_type_registry()` returns `LogicalTypeRegistry`, consistent with `get_default_type_converter()` pattern. -- `LogicalUUID.logical_type_name = "uuid.UUID"` vs `get_arrow_extension_type().extension_name = "arrow.uuid"` — intentional difference, tested in Task 4. - -### No placeholder scan - -All steps contain complete code or exact commands. No "TBD", "similar to", or "add validation" phrases. diff --git a/superpowers/plans/2026-06-14-plt-1668-logical-type-redesign.md b/superpowers/plans/2026-06-14-plt-1668-logical-type-redesign.md deleted file mode 100644 index f807fe9f3..000000000 --- a/superpowers/plans/2026-06-14-plt-1668-logical-type-redesign.md +++ /dev/null @@ -1,980 +0,0 @@ -# PLT-1668: LogicalType Redesign Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace `ExtensionTypeConverter`/`ExtensionTypeRegistry` with `LogicalType`/`LogicalTypeRegistry` so that each logical type owns its Arrow and Polars extension types directly via `get_arrow_extension_type()` / `get_polars_extension_type()`, and the registry enforces a three-way binding triplet `(logical_type_name, arrow_ext_name, python_type)`. - -**Architecture:** The `LogicalType` protocol gains two new methods (`get_arrow_extension_type`, `get_polars_extension_type`) and loses three flat properties (`extension_name`, `extension_metadata`, `storage_type`). The registry drops module-level shadow dicts entirely — uniqueness is enforced per-instance via three internal dicts. A new `make_arrow_extension_type(extension_name, storage_type, metadata) -> type[pa.ExtensionType]` helper replaces the dynamic synthesis that previously lived inside the registry. - -**Tech Stack:** Python 3.12+, PyArrow ≥ 20, Polars ≥ 1.36.0, pytest, uv. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/extension_types/protocols.py` | Rewrite | `LogicalType` protocol | -| `src/orcapod/extension_types/registry.py` | Rewrite | `make_arrow_extension_type` helper + `LogicalTypeRegistry` | -| `src/orcapod/extension_types/__init__.py` | Update | Export new names + `default_logical_type_registry` | -| `src/orcapod/extension_types/schema_walker.py` | **No change** | Self-contained; no protocol imports | -| `tests/test_extension_types/test_protocols.py` | Rewrite | Protocol conformance tests | -| `tests/test_extension_types/test_registry.py` | Rewrite | Stub helpers + all registry tests | - ---- - -### Task 1: Replace `ExtensionTypeConverter` with `LogicalType` in `protocols.py` - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py` - -- [ ] **Step 1: Overwrite `protocols.py` with the `LogicalType` protocol** - -```python -# src/orcapod/extension_types/protocols.py -"""Protocol definitions for the Arrow/Polars extension type system. - -This module defines ``LogicalType`` — the contract for all implementations -that bind a Python class to its Arrow and Polars extension type representation. - -Note: - This module is part of the parallel-build phase. The old - ``SemanticStructConverterProtocol`` in ``protocols/semantic_types_protocols.py`` - is untouched; it is removed in PLT-1660. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - - -@runtime_checkable -class LogicalType(Protocol): - """Protocol for Arrow/Polars extension-type-backed logical types. - - A ``LogicalType`` is a three-way binding between a unique logical type name - (orcapod's identifier), a Python class, and Arrow/Polars extension types. - Each implementation *owns* its Arrow and Polars extension types by providing - them directly via ``get_arrow_extension_type`` and ``get_polars_extension_type``. - - This protocol is Arrow I/O only — hashing is not a logical type responsibility. - """ - - @property - def logical_type_name(self) -> str: - """Unique orcapod identifier for this logical type. - - By convention the Python FQCN (e.g. ``"uuid.UUID"``), but any unique - string is valid. Does NOT need to match the Arrow extension type name. - """ - ... - - @property - def python_type(self) -> type: - """The Python class this logical type represents.""" - ... - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for this logical type. - - ``storage_type``, ``extension_name``, and serialised metadata are - encapsulated inside the returned type; they are no longer top-level - properties on ``LogicalType``. - - For custom types: create and return an instance of a new - ``pa.ExtensionType`` subclass (e.g. via ``make_arrow_extension_type``). - For pre-existing types: return the existing instance directly - (e.g. ``pa.uuid()``). - """ - ... - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return an instance of the Polars extension type for this logical type. - - The registry calls ``type(instance)`` to obtain the class passed to - ``pl.register_extension_type``. - """ - ... - - def python_to_storage(self, value: Any) -> Any: - """Convert a Python value to its Arrow storage representation. - - Args: - value: A Python object of type ``python_type``. - - Returns: - A value suitable for use as an Arrow scalar or array element - matching the storage type of ``get_arrow_extension_type()``. - """ - ... - - def storage_to_python(self, storage_value: Any) -> Any: - """Convert an Arrow storage value back to a Python object. - - Args: - storage_value: A scalar or array element from the Arrow storage array. - - Returns: - A Python object of type ``python_type``. - """ - ... -``` - -- [ ] **Step 2: Verify old protocol tests now fail** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: FAIL — `ExtensionTypeConverter` import error and protocol checks fail. - ---- - -### Task 2: Update `test_protocols.py` for the new `LogicalType` protocol - -**Files:** -- Modify: `tests/test_extension_types/test_protocols.py` - -- [ ] **Step 1: Overwrite `test_protocols.py`** - -```python -# tests/test_extension_types/test_protocols.py -"""Tests for LogicalType protocol.""" - -from __future__ import annotations - -import pyarrow as pa -import polars as pl - -from orcapod.extension_types.protocols import LogicalType -from orcapod.extension_types.registry import make_arrow_extension_type - - -_StubArrowExtClass = make_arrow_extension_type( - "test.module.MyType", pa.large_string(), b"test.category" -) - - -class _StubLogicalType: - """Minimal conforming implementation of LogicalType for use in tests.""" - - @property - def logical_type_name(self) -> str: - return "test.module.MyType" - - @property - def python_type(self) -> type: - return str - - def get_arrow_extension_type(self) -> pa.ExtensionType: - return _StubArrowExtClass() - - def get_polars_extension_type(self) -> pl.BaseExtension: - class _StubPL(pl.BaseExtension): - def __init__(self) -> None: - super().__init__("test.module.MyType", pl.String, None) - - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - - return _StubPL() - - def python_to_storage(self, value): - return str(value) - - def storage_to_python(self, storage_value): - return storage_value - - -def test_protocol_is_importable(): - """LogicalType can be imported from extension_types.protocols.""" - assert LogicalType is not None - - -def test_protocol_defines_required_members(): - """A conforming class is recognized as a LogicalType instance.""" - assert isinstance(_StubLogicalType(), LogicalType) - - -def test_conforming_class_satisfies_protocol(): - """A class implementing all required members works correctly via the protocol interface.""" - lt: LogicalType = _StubLogicalType() - assert lt.logical_type_name == "test.module.MyType" - assert lt.python_type is str - assert lt.get_arrow_extension_type().extension_name == "test.module.MyType" - assert isinstance(lt.get_polars_extension_type(), pl.BaseExtension) - assert lt.python_to_storage(42) == "42" - assert lt.storage_to_python("hello") == "hello" -``` - -Note: `make_arrow_extension_type` is imported from `registry.py` — this task depends on Task 3 below having the helper in place before this test file is runnable. Write the file now; run after Task 3. - ---- - -### Task 3: Add `make_arrow_extension_type` and `LogicalTypeRegistry` to `registry.py` - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` - -- [ ] **Step 1: Overwrite `registry.py` with the new implementation** - -```python -# src/orcapod/extension_types/registry.py -"""Registry for LogicalType instances. - -Registering a logical type automatically registers the corresponding -extension type in both PyArrow's and Polars' global registries. -""" - -from __future__ import annotations - -import re - -import polars as pl -import pyarrow as pa - -from orcapod.extension_types.protocols import LogicalType - - -def _sanitize(name: str) -> str: - """Replace non-alphanumeric characters with underscores. - - Used to produce a valid Python identifier for the dynamically created - ``pa.ExtensionType`` subclass name. - """ - return re.sub(r"[^A-Za-z0-9]", "_", name) - - -def make_arrow_extension_type( - extension_name: str, - storage_type: pa.DataType, - metadata: bytes | None = None, -) -> type[pa.ExtensionType]: - """Synthesise and return a ``pa.ExtensionType`` subclass. - - Returns the *class*, not an instance — callers instantiate it inside their - ``get_arrow_extension_type()`` implementation. Returning the class preserves - the option to create multiple instances or future parameterised variants from - the same class. - - This is a low-level building block. The full pattern for binding a Python - type to a specific Arrow/Polars representation — the extension type factory — - is the responsibility of each ``LogicalType`` implementation. See PLT-1656 - for the built-in implementations (``Path``, ``UPath``, ``UUID``). - - Args: - extension_name: The Arrow extension name (``ARROW:extension:name``). - storage_type: The underlying Arrow storage type. - metadata: Optional bytes stored as ``ARROW:extension:metadata``. - Defaults to ``None`` (serialised as empty bytes). - - Returns: - A ``pa.ExtensionType`` subclass. Call it with no arguments to obtain - an instance suitable for passing to ``pa.register_extension_type`` or - returning from ``get_arrow_extension_type()``. - """ - _name, _storage, _metadata = extension_name, storage_type, metadata or b"" - return type( - f"_ArrowExt_{_sanitize(extension_name)}", - (pa.ExtensionType,), - { - "__init__": lambda self: pa.ExtensionType.__init__(self, _storage, _name), - "__arrow_ext_serialize__": lambda self: _metadata, - # __arrow_ext_deserialize__ reconstructs the type descriptor from schema - # metadata (called once per IPC/Parquet read, not per value). The storage - # type and metadata are baked into the constructor via closure, so - # arguments are intentionally ignored. - "__arrow_ext_deserialize__": classmethod( - lambda cls, storage_type, serialized: cls() - ), - }, - ) - - -class LogicalTypeRegistry: - """Registry for ``LogicalType`` instances. - - Maintains a three-way binding: ``(logical_type_name, arrow_extension_name, - python_type)`` → ``LogicalType``. Each key participates in at most one - binding within a registry instance. - - Registering a logical type side-effect-registers the corresponding extension - type in PyArrow's and Polars' global registries. Pre-existing types (those - already registered externally, e.g. PyArrow's built-in ``"arrow.uuid"``) are - accepted silently — the binding is stored without error. - - The process-global ``default_logical_type_registry`` instance provides - effective process-wide uniqueness for normal use. Thread-safety is deferred. - - Example: - >>> registry = LogicalTypeRegistry() - >>> registry.register(my_logical_type) - >>> lt = registry.get_by_logical_name("uuid.UUID") - """ - - def __init__(self) -> None: - self._by_logical_name: dict[str, LogicalType] = {} - self._by_arrow_name: dict[str, LogicalType] = {} - self._by_python_type: dict[type, LogicalType] = {} - - def register(self, logical_type: LogicalType) -> None: - """Register *logical_type* and its PyArrow/Polars extension types. - - Args: - logical_type: A ``LogicalType`` instance to register. - - Raises: - ValueError: If any of the three keys (``logical_type_name``, - Arrow extension name, ``python_type``) is already bound to a - *different* ``LogicalType`` in this registry. - """ - arrow_ext_name = logical_type.get_arrow_extension_type().extension_name - py_type = logical_type.python_type - logical_name = logical_type.logical_type_name - - existing_by_logical = self._by_logical_name.get(logical_name) - existing_by_arrow = self._by_arrow_name.get(arrow_ext_name) - existing_by_python = self._by_python_type.get(py_type) - - # Triplet conflict check: raise if any key is bound to a different instance. - for existing, label, key in [ - (existing_by_logical, "logical_type_name", logical_name), - (existing_by_arrow, "arrow_extension_name", arrow_ext_name), - (existing_by_python, "python_type", py_type.__qualname__), - ]: - if existing is not None and existing is not logical_type: - raise ValueError( - f"Cannot register logical type '{logical_name}': " - f"{label} {key!r} is already bound to " - f"'{existing.logical_type_name}'." - ) - - # Idempotent check: all three keys already bound to this same instance. - if ( - existing_by_logical is logical_type - and existing_by_arrow is logical_type - and existing_by_python is logical_type - ): - return - - # Register Arrow extension type. ArrowKeyError means the name is already - # in PyArrow's global registry (pre-existing type or another registry - # instance). Accept silently — PLT-1669 adds post-error validation. - try: - pa.register_extension_type(logical_type.get_arrow_extension_type()) - except pa.lib.ArrowKeyError: - pass - - # Register Polars extension type. ValueError means already registered. - polars_ext_class = type(logical_type.get_polars_extension_type()) - try: - pl.register_extension_type(arrow_ext_name, polars_ext_class) - except ValueError: - pass - - # Store three-way binding. - self._by_logical_name[logical_name] = logical_type - self._by_arrow_name[arrow_ext_name] = logical_type - self._by_python_type[py_type] = logical_type - - def get_by_logical_name(self, name: str) -> LogicalType | None: - """Return the logical type registered under *name*, or ``None``.""" - return self._by_logical_name.get(name) - - def get_by_python_type(self, python_type: type) -> LogicalType | None: - """Return the logical type for *python_type*, or ``None``. - - Checks exact match first, then falls back to an ``issubclass`` scan. - When multiple registered types are superclasses of *python_type*, the - one registered first wins (insertion-order dict, Python 3.7+). - """ - lt = self._by_python_type.get(python_type) - if lt is not None: - return lt - for registered_type, lt in self._by_python_type.items(): - if issubclass(python_type, registered_type): - return lt - return None - - def get_by_arrow_extension_name(self, arrow_name: str) -> LogicalType | None: - """Return the logical type registered under *arrow_name*, or ``None``.""" - return self._by_arrow_name.get(arrow_name) -``` - -- [ ] **Step 2: Run protocol tests (both tasks together)** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: All 3 tests PASS. - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py \ - src/orcapod/extension_types/registry.py \ - tests/test_extension_types/test_protocols.py -git commit -m "feat(extension_types): add LogicalType protocol and LogicalTypeRegistry (PLT-1668)" -``` - ---- - -### Task 4: Rework `test_registry.py` — stubs + basic tests - -**Files:** -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Replace the imports and stub helpers at the top of `test_registry.py`** - -Replace everything from the top of the file through the `_make_stub` function definition (roughly lines 1–65 in the original) with: - -```python -"""Tests for LogicalTypeRegistry.""" - -from __future__ import annotations - -import pathlib -import tempfile -import uuid -import warnings - -import polars as pl -import pyarrow as pa -import pyarrow.parquet as pq -import pytest - -from orcapod.extension_types.protocols import LogicalType -from orcapod.extension_types.registry import LogicalTypeRegistry, make_arrow_extension_type - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _unique_name() -> str: - """Unique extension/logical name to avoid cross-test global-registry collisions.""" - return f"test.registry.{uuid.uuid4().hex[:8]}" - - -def _make_stub( - logical_name: str | None = None, - arrow_name: str | None = None, - storage: pa.DataType | None = None, - metadata: bytes | None = b"test.category", - py_type: type = str, -) -> LogicalType: - """Factory for minimal LogicalType conforming stubs. - - ``arrow_name`` defaults to ``logical_name`` when omitted. Pass separate - values to test cases that need a distinct Arrow extension name. - """ - _logical_name = logical_name or _unique_name() - _arrow_name = arrow_name or _logical_name - _storage = storage if storage is not None else pa.large_utf8() - _ArrowExt = make_arrow_extension_type(_arrow_name, _storage, metadata) - _pl_storage = pl.from_arrow(pa.array([], type=_storage)).dtype - _meta_str = metadata.decode("utf-8") if metadata else None - - class _StubPL(pl.BaseExtension): - def __init__(self) -> None: - super().__init__(_arrow_name, _pl_storage, _meta_str) - - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - - class _Stub: - @property - def logical_type_name(self) -> str: - return _logical_name - - @property - def python_type(self) -> type: - return py_type - - def get_arrow_extension_type(self) -> pa.ExtensionType: - return _ArrowExt() - - def get_polars_extension_type(self) -> pl.BaseExtension: - return _StubPL() - - def python_to_storage(self, value): - return str(value) - - def storage_to_python(self, storage_value): - return storage_value - - return _Stub() -``` - -- [ ] **Step 2: Replace all basic/lookup/PA/Polars/module-level tests (lines 70–436 in the original) with the updated equivalents** - -Remove all tests that reference removed methods (`has_extension_name`, `has_python_type`, `list_extension_names`, `list_python_types`, `get_converter_for_name`, `get_converter_for_python_type`). Replace with: - -```python -# --------------------------------------------------------------------------- -# Basic registration tests -# --------------------------------------------------------------------------- - -def test_register_stores_three_way_binding(): - """After register(), all three lookup methods return the registered LogicalType.""" - stub = _make_stub() - registry = LogicalTypeRegistry() - registry.register(stub) - - arrow_name = stub.get_arrow_extension_type().extension_name - assert registry.get_by_logical_name(stub.logical_type_name) is stub - assert registry.get_by_arrow_extension_name(arrow_name) is stub - assert registry.get_by_python_type(stub.python_type) is stub - - -def test_get_by_logical_name_miss(): - registry = LogicalTypeRegistry() - assert registry.get_by_logical_name("does.not.exist") is None - - -def test_get_by_python_type_exact(): - registry = LogicalTypeRegistry() - stub = _make_stub(py_type=bytes) - registry.register(stub) - assert registry.get_by_python_type(bytes) is stub - - -def test_get_by_python_type_subclass(): - class _Base: - pass - - class _Child(_Base): - pass - - registry = LogicalTypeRegistry() - stub = _make_stub(py_type=_Base) - registry.register(stub) - assert registry.get_by_python_type(_Child) is stub - - -def test_get_by_python_type_miss(): - registry = LogicalTypeRegistry() - assert registry.get_by_python_type(int) is None - - -def test_get_by_arrow_extension_name_miss(): - registry = LogicalTypeRegistry() - assert registry.get_by_arrow_extension_name("does.not.exist") is None - - -# --------------------------------------------------------------------------- -# Idempotency -# --------------------------------------------------------------------------- - -def test_register_idempotent_same_instance(): - """Registering the same LogicalType object twice is a no-op.""" - stub = _make_stub() - registry = LogicalTypeRegistry() - registry.register(stub) - registry.register(stub) # should not raise - assert registry.get_by_logical_name(stub.logical_type_name) is stub - - -# --------------------------------------------------------------------------- -# Triplet conflict tests -# --------------------------------------------------------------------------- - -def test_triplet_conflict_same_logical_name_raises(): - """Two LogicalTypes sharing logical_type_name -> ValueError.""" - logical_name = _unique_name() - stub1 = _make_stub(logical_name=logical_name, py_type=str) - stub2 = _make_stub(logical_name=logical_name, py_type=int) - - registry = LogicalTypeRegistry() - registry.register(stub1) - with pytest.raises(ValueError, match=logical_name): - registry.register(stub2) - - -def test_triplet_conflict_same_arrow_name_raises(): - """Two LogicalTypes sharing Arrow extension name -> ValueError.""" - shared_arrow_name = _unique_name() - stub1 = _make_stub(arrow_name=shared_arrow_name, py_type=str) - stub2 = _make_stub(arrow_name=shared_arrow_name, py_type=int) - - registry = LogicalTypeRegistry() - registry.register(stub1) - with pytest.raises(ValueError, match=shared_arrow_name): - registry.register(stub2) - - -def test_triplet_conflict_same_python_type_raises(): - """Two LogicalTypes sharing python_type -> ValueError.""" - stub1 = _make_stub(py_type=float) - stub2 = _make_stub(py_type=float) - - registry = LogicalTypeRegistry() - registry.register(stub1) - with pytest.raises(ValueError, match="float"): - registry.register(stub2) - - -# --------------------------------------------------------------------------- -# Pre-existing type tolerance tests -# --------------------------------------------------------------------------- - -def test_register_preexisting_arrow_type_succeeds(): - """ArrowKeyError from PA global registry is accepted silently; binding is stored.""" - name = _unique_name() - - class _ExternalPA(pa.ExtensionType): - def __init__(self) -> None: - pa.ExtensionType.__init__(self, pa.large_utf8(), name) - - def __arrow_ext_serialize__(self): - return b"" - - @classmethod - def __arrow_ext_deserialize__(cls, st, se): - return cls() - - pa.register_extension_type(_ExternalPA()) # pre-register externally - - stub = _make_stub(arrow_name=name) - registry = LogicalTypeRegistry() - registry.register(stub) # must not raise - - assert registry.get_by_logical_name(stub.logical_type_name) is stub - assert registry.get_by_arrow_extension_name(name) is stub - assert registry.get_by_python_type(stub.python_type) is stub - - -def test_register_preexisting_polars_type_succeeds(): - """ValueError from Polars global registry is accepted silently; binding is stored.""" - name = _unique_name() - - # Pre-register in PA first to avoid PA-level conflict - class _ExternalPA(pa.ExtensionType): - def __init__(self) -> None: - pa.ExtensionType.__init__(self, pa.large_utf8(), name) - - def __arrow_ext_serialize__(self): - return b"" - - @classmethod - def __arrow_ext_deserialize__(cls, st, se): - return cls() - - pa.register_extension_type(_ExternalPA()) - - class _ExternalPL(pl.BaseExtension): - def __init__(self) -> None: - super().__init__(name, pl.String, None) - - @classmethod - def ext_from_params(cls, n, s, m): - return cls() - - pl.register_extension_type(name, _ExternalPL) - - stub = _make_stub(arrow_name=name) - registry = LogicalTypeRegistry() - registry.register(stub) # must not raise - - assert registry.get_by_logical_name(stub.logical_type_name) is stub - assert registry.get_by_arrow_extension_name(name) is stub - assert registry.get_by_python_type(stub.python_type) is stub - - -# --------------------------------------------------------------------------- -# PyArrow global registry: our type gets registered -# --------------------------------------------------------------------------- - -def test_register_populates_arrow_global_registry(): - """After register(), PA global registry contains the extension type.""" - stub = _make_stub() - registry = LogicalTypeRegistry() - registry.register(stub) - - arrow_name = stub.get_arrow_extension_type().extension_name - - class _Probe(pa.ExtensionType): - def __init__(self) -> None: - pa.ExtensionType.__init__(self, pa.large_utf8(), arrow_name) - - def __arrow_ext_serialize__(self): - return b"" - - @classmethod - def __arrow_ext_deserialize__(cls, st, se): - return cls() - - with pytest.raises(pa.lib.ArrowKeyError): - pa.register_extension_type(_Probe()) -``` - -- [ ] **Step 3: Run the basic + idempotency + triplet + pre-existing tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -k "not round_trip and not parquet and not module_instance" -``` - -Expected: All newly written tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_extension_types/test_registry.py -git commit -m "test(extension_types): rework test_registry for LogicalTypeRegistry (PLT-1668)" -``` - ---- - -### Task 5: Update end-to-end tests in `test_registry.py` - -**Files:** -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Replace the `_Color`, `_make_color_converter`, `_build_ext_array`, and end-to-end test functions** - -Remove the old `_Color` / `_make_color_converter` / `_build_ext_array` block and the three round-trip tests. Replace with: - -```python -# --------------------------------------------------------------------------- -# End-to-end helpers -# --------------------------------------------------------------------------- - -class _Color: - """Minimal Python class used to exercise the logical type contract end-to-end.""" - - def __init__(self, hex_str: str) -> None: - self.hex_str = hex_str - - def __eq__(self, other: object) -> bool: - return isinstance(other, _Color) and self.hex_str == other.hex_str - - def __repr__(self) -> str: - return f"Color({self.hex_str!r})" - - -def _make_color_logical_type() -> LogicalType: - """LogicalType for _Color, backed by pa.large_utf8() storage.""" - _name = _unique_name() - _ArrowExt = make_arrow_extension_type(_name, pa.large_utf8(), b"test.color") - _pl_storage = pl.from_arrow(pa.array([], type=pa.large_utf8())).dtype - - class _ColorPL(pl.BaseExtension): - def __init__(self) -> None: - super().__init__(_name, _pl_storage, "test.color") - - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - - class _ColorLogicalType: - @property - def logical_type_name(self) -> str: - return _name - - @property - def python_type(self) -> type: - return _Color - - def get_arrow_extension_type(self) -> pa.ExtensionType: - return _ArrowExt() - - def get_polars_extension_type(self) -> pl.BaseExtension: - return _ColorPL() - - def python_to_storage(self, value: _Color) -> str: - return value.hex_str - - def storage_to_python(self, storage_value: str) -> _Color: - return _Color(storage_value) - - return _ColorLogicalType() - - -def _build_ext_array(lt: LogicalType, values: list) -> pa.Array: - """Build a PA extension array from Python values using the logical type.""" - arrow_ext = lt.get_arrow_extension_type() - storage_values = [lt.python_to_storage(v) for v in values] - storage_arr = pa.array(storage_values, type=arrow_ext.storage_type) - return storage_arr.cast(arrow_ext) - - -# --------------------------------------------------------------------------- -# End-to-end integration tests -# --------------------------------------------------------------------------- - -def test_python_class_round_trip(): - """Python objects -> Arrow extension array -> Python objects via logical type methods.""" - lt = _make_color_logical_type() - registry = LogicalTypeRegistry() - registry.register(lt) - - originals = [_Color("#ff0000"), _Color("#00ff00"), _Color("#0000ff")] - ext_arr = _build_ext_array(lt, originals) - - recovered = [lt.storage_to_python(v.as_py()) for v in ext_arr.storage] - assert recovered == originals - - -def test_arrow_polars_round_trip(): - """PA ext array -> pl.from_arrow -> to_arrow() preserves extension type and values.""" - lt = _make_color_logical_type() - registry = LogicalTypeRegistry() - registry.register(lt) - - originals = [_Color("#aabbcc"), _Color("#112233")] - ext_arr = _build_ext_array(lt, originals) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - pl_series = pl.from_arrow(ext_arr) - - arrow_name = lt.get_arrow_extension_type().extension_name - assert isinstance(pl_series.dtype, pl.BaseExtension) - assert pl_series.dtype.ext_name() == arrow_name - - arr_back = pl_series.to_arrow() - assert arr_back.type.extension_name == arrow_name - - recovered = [lt.storage_to_python(v.as_py()) for v in arr_back.storage] - assert recovered == originals - - -def test_parquet_round_trip(): - """PA ext array -> Parquet -> read back; extension type and values preserved.""" - lt = _make_color_logical_type() - registry = LogicalTypeRegistry() - registry.register(lt) - - originals = [_Color("#deadbe"), _Color("#cafeba")] - ext_arr = _build_ext_array(lt, originals) - schema = pa.schema([pa.field("color", ext_arr.type), pa.field("id", pa.int32())]) - table = pa.table( - {"color": ext_arr, "id": pa.array([1, 2], type=pa.int32())}, - schema=schema, - ) - - with tempfile.TemporaryDirectory() as tmp: - path = pathlib.Path(tmp) / "test.parquet" - pq.write_table(table, path) - table_back = pq.read_table(path) - - arrow_name = lt.get_arrow_extension_type().extension_name - assert table_back.schema.field("color").type.extension_name == arrow_name - storage_arr = table_back.column("color").combine_chunks().storage - recovered = [lt.storage_to_python(v.as_py()) for v in storage_arr] - assert recovered == originals - - -# --------------------------------------------------------------------------- -# Module-level instance test -# --------------------------------------------------------------------------- - -def test_logical_type_registry_module_instance(): - """extension_types.default_logical_type_registry is a LogicalTypeRegistry, starts empty.""" - from orcapod import extension_types - - assert isinstance(extension_types.default_logical_type_registry, LogicalTypeRegistry) - # PLT-1668 scope: no built-in logical types registered yet (that is PLT-1656). - assert extension_types.default_logical_type_registry.get_by_logical_name("uuid.UUID") is None -``` - -- [ ] **Step 2: Run all registry tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: All tests PASS. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_registry.py -git commit -m "test(extension_types): add end-to-end and module-instance tests for LogicalTypeRegistry (PLT-1668)" -``` - ---- - -### Task 6: Update `__init__.py` exports - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` - -- [ ] **Step 1: Overwrite `__init__.py`** - -```python -# src/orcapod/extension_types/__init__.py -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for logical types that bind -Python classes to their Arrow and Polars extension type representation. - -The module-level ``default_logical_type_registry`` instance is the process default. -Built-in registrations (``Path``, ``UPath``, ``UUID``) are added by PLT-1656. -``DataContext`` wiring is added by PLT-1660. -""" - -from __future__ import annotations - -from .protocols import LogicalType -from .registry import LogicalTypeRegistry, make_arrow_extension_type -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema - -default_logical_type_registry = LogicalTypeRegistry() - -__all__ = [ - "LogicalType", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "default_logical_type_registry", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", -] -``` - -- [ ] **Step 2: Run the full `test_extension_types` suite** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: All tests in `test_protocols.py`, `test_registry.py`, and `test_schema_walker.py` PASS. - -- [ ] **Step 3: Run the complete test suite to catch any regressions** - -```bash -uv run pytest --tb=short -q -``` - -Expected: All tests pass. No references to `ExtensionTypeConverter`, `ExtensionTypeRegistry`, or `default_extension_type_registry` remain outside of the deleted/replaced files. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "feat(extension_types): update __init__ exports for LogicalType redesign (PLT-1668)" -``` - ---- - -## Self-Review Checklist - -After completing all tasks, verify: - -- [ ] `LogicalType` has exactly 6 members: `logical_type_name`, `python_type`, `get_arrow_extension_type`, `get_polars_extension_type`, `python_to_storage`, `storage_to_python` -- [ ] `LogicalTypeRegistry` has exactly 3 lookup methods: `get_by_logical_name`, `get_by_python_type`, `get_by_arrow_extension_name` -- [ ] No reference to `ExtensionTypeConverter`, `ExtensionTypeRegistry`, `default_extension_type_registry`, `_ARROW_REGISTRY`, `_POLARS_REGISTRY`, `_register_arrow_ext_type`, or `_register_polars_ext_type` remains anywhere in `src/` or `tests/` -- [ ] `make_arrow_extension_type` returns `type[pa.ExtensionType]` (a class, not an instance) -- [ ] Triplet conflict error messages include the conflicting key name so `pytest.raises(ValueError, match=)` works -- [ ] Pre-existing-type tests pre-register externally then call `registry.register()` — the call must not raise -- [ ] `test_schema_walker.py` still passes unchanged diff --git a/superpowers/plans/2026-06-15-plt-1670-orcapod-namespace-builtin-types.md b/superpowers/plans/2026-06-15-plt-1670-orcapod-namespace-builtin-types.md deleted file mode 100644 index 4f10b9ee5..000000000 --- a/superpowers/plans/2026-06-15-plt-1670-orcapod-namespace-builtin-types.md +++ /dev/null @@ -1,721 +0,0 @@ -# PLT-1670: Namespace Built-in Extension Types under `orcapod.*` Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Rename the three built-in Arrow extension types from upstream module-path names (`"pathlib.Path"`, `"upath.UPath"`, `"uuid.UUID"`) to Orcapod-owned namespaced names (`"orcapod.path"`, `"orcapod.upath"`, `"orcapod.uuid"`), and expose `Path`, `UPath`, `UUID` type aliases at the top-level `orcapod` namespace. - -**Architecture:** The three `LogicalType` classes in `builtin_logical_types.py` each carry a string constant used as both `logical_type_name` and the Arrow/Polars extension name — changing those constants is the entirety of the rename. Top-level aliases are simple re-exports in `__init__.py` that expose the upstream types under an Orcapod-stable symbol. Tests are updated last to match reality after the TDD red → green cycle. - -**Tech Stack:** Python, PyArrow (extension types), Polars (extension types), pytest via `uv run pytest`. - ---- - -## File Map - -| File | Action | What changes | -|------|--------|-------------| -| `tests/test_extension_types/test_builtin_logical_types.py` | Modify | Update 13 string assertions from old extension names to new `orcapod.*` names; add 5 alias tests | -| `src/orcapod/extension_types/builtin_logical_types.py` | Modify | Rename 6 string constants (2 per class: `_arrow_ext_class`, `_polars_ext_class`), 3 `logical_type_name` class attributes, update module and class docstrings | -| `src/orcapod/__init__.py` | Modify | Add `Path`, `UPath`, `UUID` re-exports with stability docstring; add to `__all__` | - -No other files need to change — the context config (`contexts/data/v0.1.json`) references classes by dotted name, not by extension name string, so it is unaffected. - ---- - -## Task 1: Update tests to assert on new `orcapod.*` extension names - -This is the TDD red step. After this task the test suite will fail with assertion errors until Task 2 fixes the implementation. - -**Files:** -- Modify: `tests/test_extension_types/test_builtin_logical_types.py` - -- [ ] **Step 1: Run the current test suite to confirm it is green before any changes** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all tests pass (green baseline). - -- [ ] **Step 2: Update `test_logical_path_logical_type_name` (line 31)** - -Change: -```python -assert LogicalPath().logical_type_name == "pathlib.Path" -``` -To: -```python -assert LogicalPath().logical_type_name == "orcapod.path" -``` - -- [ ] **Step 3: Update `test_logical_path_arrow_ext_name` (line 44)** - -Change: -```python -assert LogicalPath().get_arrow_extension_type().extension_name == "pathlib.Path" -``` -To: -```python -assert LogicalPath().get_arrow_extension_type().extension_name == "orcapod.path" -``` - -- [ ] **Step 4: Update `test_logical_upath_logical_type_name` (line 103)** - -Change: -```python -assert LogicalUPath().logical_type_name == "upath.UPath" -``` -To: -```python -assert LogicalUPath().logical_type_name == "orcapod.upath" -``` - -- [ ] **Step 5: Update `test_logical_upath_arrow_ext_name` (line 116)** - -Change: -```python -assert LogicalUPath().get_arrow_extension_type().extension_name == "upath.UPath" -``` -To: -```python -assert LogicalUPath().get_arrow_extension_type().extension_name == "orcapod.upath" -``` - -- [ ] **Step 6: Update `test_logical_uuid_logical_type_name` (line 173)** - -Change: -```python -assert LogicalUUID().logical_type_name == "uuid.UUID" -``` -To: -```python -assert LogicalUUID().logical_type_name == "orcapod.uuid" -``` - -- [ ] **Step 7: Update `test_logical_uuid_arrow_ext_name` (lines 187–188)** - -Change: -```python - assert lt.get_arrow_extension_type().extension_name == "uuid.UUID" - assert lt.get_arrow_extension_type().extension_name == lt.logical_type_name -``` -To: -```python - assert lt.get_arrow_extension_type().extension_name == "orcapod.uuid" - assert lt.get_arrow_extension_type().extension_name == lt.logical_type_name -``` - -- [ ] **Step 8: Update `test_logical_uuid_registration_does_not_raise` (lines 248–249)** - -Change: -```python - assert registry.get_by_logical_name("uuid.UUID") is lt - assert registry.get_by_arrow_extension_name("uuid.UUID") is lt -``` -To: -```python - assert registry.get_by_logical_name("orcapod.uuid") is lt - assert registry.get_by_arrow_extension_name("orcapod.uuid") is lt -``` - -- [ ] **Step 9: Update default-context tests — `test_default_context_registry_has_logical_path` (line 384)** - -Change: -```python - lt = registry.get_by_logical_name("pathlib.Path") -``` -To: -```python - lt = registry.get_by_logical_name("orcapod.path") -``` - -- [ ] **Step 10: Update `test_default_context_registry_lookup_by_arrow_name_path` (line 404)** - -Change: -```python - lt = registry.get_by_arrow_extension_name("pathlib.Path") -``` -To: -```python - lt = registry.get_by_arrow_extension_name("orcapod.path") -``` - -- [ ] **Step 11: Update `test_default_context_registry_has_logical_upath` (line 414)** - -Change: -```python - lt = registry.get_by_logical_name("upath.UPath") -``` -To: -```python - lt = registry.get_by_logical_name("orcapod.upath") -``` - -- [ ] **Step 12: Update `test_default_context_registry_has_logical_uuid` (line 434)** - -Change: -```python - lt = registry.get_by_logical_name("uuid.UUID") -``` -To: -```python - lt = registry.get_by_logical_name("orcapod.uuid") -``` - -- [ ] **Step 13: Update `test_default_context_registry_lookup_by_arrow_name_uuid` (line 444)** - -Change: -```python - lt = registry.get_by_arrow_extension_name("uuid.UUID") -``` -To: -```python - lt = registry.get_by_arrow_extension_name("orcapod.uuid") -``` - -- [ ] **Step 14: Run tests to confirm they are now red** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v --tb=line 2>&1 | grep -E "FAILED|PASSED|ERROR" | head -30 -``` - -Expected: 13 tests now fail with `AssertionError`; all others pass. - ---- - -## Task 2: Rename extension type strings in `builtin_logical_types.py` - -This makes the red tests green. All 6 extension-name string constants, all 3 `logical_type_name` class attributes, and docstrings are updated to the `orcapod.*` namespace. - -**Files:** -- Modify: `src/orcapod/extension_types/builtin_logical_types.py` - -- [ ] **Step 1: Update the module-level docstring** - -Change the opening docstring lines 6–8 from: -```python -- ``LogicalPath``: maps ``pathlib.Path`` ↔ Arrow large_string extension "pathlib.Path" -- ``LogicalUPath``: maps ``upath.UPath`` ↔ Arrow large_string extension "upath.UPath" -- ``LogicalUUID``: maps ``uuid.UUID`` ↔ Arrow large_binary extension "uuid.UUID" -``` -To: -```python -- ``LogicalPath``: maps ``pathlib.Path`` ↔ Arrow large_string extension ``"orcapod.path"`` -- ``LogicalUPath``: maps ``upath.UPath`` ↔ Arrow large_string extension ``"orcapod.upath"`` -- ``LogicalUUID``: maps ``uuid.UUID`` ↔ Arrow large_binary extension ``"orcapod.uuid"`` -``` - -And replace the full module docstring with the updated version that adds the stability rationale note: - -```python -"""Built-in LogicalType implementations for orcapod. - -Provides three built-in logical types registered into the default -``DataContext.logical_type_registry`` via ``contexts/data/v0.1.json``: - -- ``LogicalPath``: maps ``pathlib.Path`` ↔ Arrow large_string extension ``"orcapod.path"`` -- ``LogicalUPath``: maps ``upath.UPath`` ↔ Arrow large_string extension ``"orcapod.upath"`` -- ``LogicalUUID``: maps ``uuid.UUID`` ↔ Arrow large_binary extension ``"orcapod.uuid"`` - -All three types use the ``orcapod.*`` extension name namespace rather than the upstream -module-qualified names (``"pathlib.Path"``, etc.). This gives Orcapod stable ownership of -the on-disk extension identity: even if the upstream library is renamed or restructured, -data written with these extension names continues to be readable without modification. - -Note: - All imports from orcapod.extension_types use direct submodule paths - (e.g. ``from orcapod.extension_types.registry import ...``) rather than - the package ``__init__`` to avoid circular imports when the context system - loads this module at startup. -""" -``` - -- [ ] **Step 2: Update `LogicalPath` class — class attributes and docstrings** - -Replace the `LogicalPath` class definition (lines 30–94) with: - -```python -class LogicalPath: - """Logical type for ``pathlib.Path``. - - Stores paths as Arrow large strings using the custom extension type - ``"orcapod.path"``. - - The extension name ``"orcapod.path"`` is Orcapod-owned and stable; it does not - depend on the upstream ``pathlib`` module path. Use ``orcapod.Path`` (a top-level - alias for ``pathlib.Path``) as the preferred way to reference this type in user code. - - Example: - >>> lt = LogicalPath() - >>> lt.python_to_storage(pathlib.Path("/tmp/foo")) - '/tmp/foo' - >>> lt.storage_to_python('/tmp/foo') - PosixPath('/tmp/foo') - """ - - _arrow_ext_class = make_arrow_extension_type("orcapod.path", pa.large_string()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("orcapod.path", pa.large_string()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "orcapod.path" - python_type: type = pathlib.Path - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for ``pathlib.Path``. - - Returns: - A cached ``pa.ExtensionType`` instance with extension name - ``"orcapod.path"`` and storage type ``pa.large_string()``. - """ - if LogicalPath._arrow_ext is None: - LogicalPath._arrow_ext = LogicalPath._arrow_ext_class() - return LogicalPath._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``pathlib.Path``. - - Returns: - A cached ``pl.BaseExtension`` instance registered under - ``"orcapod.path"``. - """ - if LogicalPath._polars_ext is None: - LogicalPath._polars_ext = LogicalPath._polars_ext_class() - return LogicalPath._polars_ext - - def python_to_storage(self, value: Any) -> str: - """Convert a ``pathlib.Path`` to its string representation. - - Args: - value: A ``pathlib.Path`` instance. - - Returns: - The string form of the path (e.g. ``"/tmp/foo"``). - """ - return str(value) - - def storage_to_python(self, storage_value: Any) -> pathlib.Path: - """Reconstruct a ``pathlib.Path`` from its string representation. - - Args: - storage_value: A string path as stored in Arrow. - - Returns: - A ``pathlib.Path`` instance. - """ - return pathlib.Path(storage_value) -``` - -- [ ] **Step 3: Update `LogicalUPath` class — class attributes and docstrings** - -Replace the `LogicalUPath` class definition (lines 97–161) with: - -```python -class LogicalUPath: - """Logical type for ``upath.UPath``. - - Stores paths as Arrow large strings using the custom extension type - ``"orcapod.upath"``. - - The extension name ``"orcapod.upath"`` is Orcapod-owned and stable; it does not - depend on the upstream ``upath`` module path. Use ``orcapod.UPath`` (a top-level - alias for ``upath.UPath``) as the preferred way to reference this type in user code. - - Example: - >>> lt = LogicalUPath() - >>> lt.python_to_storage(UPath("s3://bucket/key")) - 's3://bucket/key' - >>> lt.storage_to_python("s3://bucket/key") - UPath('s3://bucket/key') - """ - - _arrow_ext_class = make_arrow_extension_type("orcapod.upath", pa.large_string()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("orcapod.upath", pa.large_string()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "orcapod.upath" - python_type: type = UPath - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for ``upath.UPath``. - - Returns: - A cached ``pa.ExtensionType`` instance with extension name - ``"orcapod.upath"`` and storage type ``pa.large_string()``. - """ - if LogicalUPath._arrow_ext is None: - LogicalUPath._arrow_ext = LogicalUPath._arrow_ext_class() - return LogicalUPath._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``upath.UPath``. - - Returns: - A cached ``pl.BaseExtension`` instance registered under - ``"orcapod.upath"``. - """ - if LogicalUPath._polars_ext is None: - LogicalUPath._polars_ext = LogicalUPath._polars_ext_class() - return LogicalUPath._polars_ext - - def python_to_storage(self, value: Any) -> str: - """Convert a ``upath.UPath`` to its string representation. - - Args: - value: A ``upath.UPath`` instance. - - Returns: - The string form of the path (e.g. ``"s3://bucket/key"``). - """ - return str(value) - - def storage_to_python(self, storage_value: Any) -> UPath: - """Reconstruct a ``upath.UPath`` from its string representation. - - Args: - storage_value: A string path as stored in Arrow. - - Returns: - A ``upath.UPath`` instance. - """ - return UPath(storage_value) -``` - -- [ ] **Step 4: Update `LogicalUUID` class — class attributes and docstrings** - -Replace the `LogicalUUID` class definition (lines 164–236) with: - -```python -class LogicalUUID: - """Logical type for ``uuid.UUID``. - - Stores UUIDs as Arrow binary (16 bytes) using the custom extension type - ``"orcapod.uuid"``. Both the Arrow extension name and ``logical_type_name`` - are ``"orcapod.uuid"``, consistent with ``LogicalPath`` and ``LogicalUPath``. - - The extension name ``"orcapod.uuid"`` is Orcapod-owned and stable, replacing - the previous ``"uuid.UUID"`` name that mirrored PyArrow's ``"arrow.uuid"`` - territory. Use ``orcapod.UUID`` (a top-level alias for ``uuid.UUID``) as the - preferred way to reference this type in user code. - - The storage type is ``pa.large_binary()`` (variable-length binary), using - big-endian byte order as returned by ``uuid.UUID.bytes``. ``large_binary`` - is used rather than ``pa.binary(16)`` (fixed-size) because Polars maps - fixed-size binary to variable-length on the round-trip, which would - conflict with the deserializer's storage type check. - - Example: - >>> import uuid - >>> lt = LogicalUUID() - >>> u = uuid.uuid4() - >>> lt.storage_to_python(lt.python_to_storage(u)) == u - True - """ - - _arrow_ext_class = make_arrow_extension_type("orcapod.uuid", pa.large_binary()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("orcapod.uuid", pa.large_binary()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "orcapod.uuid" - python_type: type = _uuid_module.UUID - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for ``uuid.UUID``. - - Returns: - A cached ``pa.ExtensionType`` instance with extension name - ``"orcapod.uuid"`` and storage type ``pa.large_binary()``. - """ - if LogicalUUID._arrow_ext is None: - LogicalUUID._arrow_ext = LogicalUUID._arrow_ext_class() - return LogicalUUID._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``uuid.UUID``. - - Returns: - A cached ``pl.BaseExtension`` instance registered under - ``"orcapod.uuid"``. - """ - if LogicalUUID._polars_ext is None: - LogicalUUID._polars_ext = LogicalUUID._polars_ext_class() - return LogicalUUID._polars_ext - - def python_to_storage(self, value: Any) -> bytes: - """Convert a ``uuid.UUID`` to its 16-byte binary representation. - - Args: - value: A ``uuid.UUID`` instance. - - Returns: - A 16-byte ``bytes`` object (big-endian byte order, as per - ``uuid.UUID.bytes``). - """ - return value.bytes - - def storage_to_python(self, storage_value: Any) -> _uuid_module.UUID: - """Reconstruct a ``uuid.UUID`` from its 16-byte binary representation. - - Args: - storage_value: A bytes-like object of length 16. - - Returns: - A ``uuid.UUID`` instance. - """ - return _uuid_module.UUID(bytes=bytes(storage_value)) -``` - -- [ ] **Step 5: Run the failing tests to confirm they are now green** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/builtin_logical_types.py \ - tests/test_extension_types/test_builtin_logical_types.py -git commit -m "feat(extension_types): rename built-in extension types to orcapod.* namespace - -LogicalPath: 'pathlib.Path' -> 'orcapod.path' -LogicalUPath: 'upath.UPath' -> 'orcapod.upath' -LogicalUUID: 'uuid.UUID' -> 'orcapod.uuid' - -Orcapod now owns the canonical extension identity for all three built-in -types, decoupling on-disk names from upstream library module paths." -``` - ---- - -## Task 3: Add tests for top-level `orcapod.Path`, `orcapod.UPath`, `orcapod.UUID` aliases - -TDD red step for the alias feature. These tests will fail until Task 4 adds the aliases. - -**Files:** -- Modify: `tests/test_extension_types/test_builtin_logical_types.py` - -- [ ] **Step 1: Append the alias test block at the end of the test file** - -Add the following to the end of `tests/test_extension_types/test_builtin_logical_types.py`: - -```python -# --------------------------------------------------------------------------- -# Top-level orcapod namespace alias tests -# --------------------------------------------------------------------------- - - -def test_orcapod_path_alias_is_pathlib_path(): - """orcapod.Path is the same object as pathlib.Path.""" - import pathlib - - import orcapod - - assert orcapod.Path is pathlib.Path - - -def test_orcapod_upath_alias_is_upath_upath(): - """orcapod.UPath is the same object as upath.UPath.""" - from upath import UPath - - import orcapod - - assert orcapod.UPath is UPath - - -def test_orcapod_uuid_alias_is_uuid_uuid(): - """orcapod.UUID is the same object as uuid.UUID.""" - import uuid - - import orcapod - - assert orcapod.UUID is uuid.UUID - - -def test_orcapod_path_alias_in_all(): - """orcapod.Path appears in orcapod.__all__.""" - import orcapod - - assert "Path" in orcapod.__all__ - - -def test_orcapod_upath_alias_in_all(): - """orcapod.UPath appears in orcapod.__all__.""" - import orcapod - - assert "UPath" in orcapod.__all__ - - -def test_orcapod_uuid_alias_in_all(): - """orcapod.UUID appears in orcapod.__all__.""" - import orcapod - - assert "UUID" in orcapod.__all__ -``` - -- [ ] **Step 2: Run the new tests to confirm they are red** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v -k "alias" --tb=short 2>&1 -``` - -Expected: 6 tests fail with `AttributeError: module 'orcapod' has no attribute 'Path'` (or similar). - ---- - -## Task 4: Add `Path`, `UPath`, `UUID` aliases to `src/orcapod/__init__.py` - -**Files:** -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Add the alias imports and `__all__` entries** - -Replace the entire content of `src/orcapod/__init__.py` with: - -```python -from .config import ( - DEFAULT_CONFIG, - DisplayConfig, - HashingConfig, - OrcapodConfig, - load_config, -) -from .core.function_pod import ( - FunctionPod, - function_pod, -) -from .core.nodes.source_node import SourceNode -from .pipeline import Pipeline, PipelineJob -from .semantic_types.dataclass_encoding import register_dataclass - -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -# Stable type aliases — preferred over importing directly from pathlib/upath/uuid. -# -# These aliases are the recommended way to reference these types in orcapod user code. -# Even if an upstream library is renamed or restructured, these symbols remain stable -# at ``orcapod.Path``, ``orcapod.UPath``, and ``orcapod.UUID``. Their Arrow extension -# types are registered under the ``orcapod.*`` namespace (``"orcapod.path"``, -# ``"orcapod.upath"``, ``"orcapod.uuid"``), so on-disk identity is also decoupled -# from upstream module paths. -from pathlib import Path -from upath import UPath -from uuid import UUID - -__all__ = [ - "DEFAULT_CONFIG", - "DisplayConfig", - "HashingConfig", - "OrcapodConfig", - "load_config", - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceNode", - "register_dataclass", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", - # Stable type aliases - "Path", - "UPath", - "UUID", -] -``` - -- [ ] **Step 2: Run the alias tests to confirm they are now green** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v -k "alias" --tb=short 2>&1 -``` - -Expected: all 6 alias tests pass. - -- [ ] **Step 3: Run the full builtin logical types test suite** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all tests pass (the full suite, not just alias tests). - -- [ ] **Step 4: Run the broader extension_types test suite to check for regressions** - -```bash -uv run pytest tests/test_extension_types/ -v --tb=short 2>&1 | tail -30 -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/__init__.py \ - tests/test_extension_types/test_builtin_logical_types.py -git commit -m "feat(orcapod): expose Path, UPath, UUID as stable top-level aliases - -Adds orcapod.Path, orcapod.UPath, orcapod.UUID as re-exports of -pathlib.Path, upath.UPath, and uuid.UUID respectively. These are the -preferred symbols for user code — stable even if upstream libraries -rename their types or module paths." -``` - ---- - -## Task 5: Final verification — full test suite - -- [ ] **Step 1: Run the complete test suite** - -```bash -uv run pytest tests/ -x --tb=short 2>&1 | tail -40 -``` - -Expected: all tests pass (no regressions in any other test module). - -- [ ] **Step 2: Verify the branch is clean and ready for PR** - -```bash -git status -git log --oneline origin/extension-type-system..HEAD -``` - -Expected: 2 commits ahead of `extension-type-system`, working tree clean. - ---- - -## Self-Review Checklist - -**Spec coverage:** - -| Requirement | Task that covers it | -|-------------|-------------------| -| `LogicalPath` registers under `"orcapod.path"` | Task 2 Step 2 | -| `LogicalUPath` registers under `"orcapod.upath"` | Task 2 Step 3 | -| `LogicalUUID` registers under `"orcapod.uuid"` | Task 2 Step 4 | -| `orcapod.uuid` no longer conflicts with `arrow.uuid` | Task 2 Step 4 (new name `"orcapod.uuid"` vs PyArrow's `"arrow.uuid"`) | -| `orcapod.Path` alias exposed at top-level | Task 4 Step 1 | -| `orcapod.UPath` alias exposed at top-level | Task 4 Step 1 | -| `orcapod.UUID` alias exposed at top-level | Task 4 Step 1 | -| Aliases documented as preferred + stability rationale | Task 4 Step 1 (comment block) | -| Stability rationale in module docstring | Task 2 Step 1 | -| Existing round-trip behavior continues to work | Task 5 Step 1 | -| Unit tests updated to assert `orcapod.*` names | Task 1 + Task 3 | - -**No placeholders:** All steps contain exact code. No "TBD" or "similar to above" references. - -**Type consistency:** `logical_type_name` constants and `extension_name` strings are consistent across Tasks 1, 2, 3, and 4 — `"orcapod.path"`, `"orcapod.upath"`, `"orcapod.uuid"` throughout. diff --git a/superpowers/plans/2026-06-15-plt-1672-write-side-logical-type-factory.md b/superpowers/plans/2026-06-15-plt-1672-write-side-logical-type-factory.md deleted file mode 100644 index 931e78c70..000000000 --- a/superpowers/plans/2026-06-15-plt-1672-write-side-logical-type-factory.md +++ /dev/null @@ -1,1482 +0,0 @@ -# PLT-1672: Write-Side Logical Type Factory Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add Python-class-keyed write-side factory dispatch to `LogicalTypeRegistry` and wire it into `UniversalTypeConverter` and `_FunctionPodBase` so that unregistered Python types are auto-registered via a factory at function pod declaration time. - -**Architecture:** Two new factory dispatch axes (category-keyed for reads, python-class-keyed for writes) are unified in `LogicalTypeRegistry`'s `ensure_logical_type_for_python_class` with a shared MRO resolution algorithm. A recursive `_extract_leaf_classes` unwrapper in a new `type_utils.py` feeds the write-side trigger in `_FunctionPodBase.__init__`. `UniversalTypeConverter` is extended with a one-line priority check so registered extension types take precedence over the old shape-based system at encoding time. - -**Tech Stack:** Python 3.12+, PyArrow, Polars, `typing.get_origin`/`get_args` for generic annotation unwrapping. All tests via `uv run pytest`. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/extension_types/protocols.py` | Modify | Rename `create_logical_type` → `reconstruct_from_arrow`; add `create_for_python_type` | -| `src/orcapod/extension_types/registry.py` | Modify | Rename `_factories` → `_category_factories`; add `_python_class_factories`; extend `register_logical_type_factory`; add `ensure_logical_type_for_python_class` | -| `src/orcapod/extension_types/type_utils.py` | Create | `_extract_leaf_classes(annotation)` — recursive generic annotation unwrapper | -| `src/orcapod/extension_types/__init__.py` | Modify | Export `_extract_leaf_classes` | -| `src/orcapod/semantic_types/universal_converter.py` | Modify | Add `_logical_type_registry` attribute; insert priority check before `semantic_registry` in `_convert_python_to_arrow` | -| `src/orcapod/contexts/core.py` | Modify | Add `DataContext.__post_init__` to wire `logical_type_registry` into `type_converter` | -| `src/orcapod/core/function_pod.py` | Modify | Add `_ARROW_NATIVE_TYPES`, `_trigger_write_side_registration`; call from `_FunctionPodBase.__init__` | -| `tests/test_extension_types/test_protocols.py` | Modify | Update `_StubFactory` stub; add `create_for_python_type` conformance test | -| `tests/test_extension_types/test_registry.py` | Modify | Update all `register_logical_type_factory` call sites; add `ensure_logical_type_for_python_class` tests | -| `tests/test_extension_types/test_type_utils.py` | Create | Tests for `_extract_leaf_classes` | -| `tests/test_semantic_types/test_universal_converter.py` | Modify | Add `_logical_type_registry` priority check tests | -| `tests/test_core/function_pod/test_write_side_registration.py` | Create | End-to-end pod-declaration trigger tests | - ---- - -## Task 1: Rename `create_logical_type` → `reconstruct_from_arrow` in `LogicalTypeFactoryProtocol` - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py` -- Modify: `src/orcapod/extension_types/registry.py` (call site) -- Modify: `tests/test_extension_types/test_protocols.py` -- Modify: `tests/test_extension_types/test_registry.py` (all uses of `create_logical_type`) - -- [ ] **Step 1: Update `_StubFactory` in test_protocols.py to use the new name** - -Edit `tests/test_extension_types/test_protocols.py`. Replace the `_StubFactory` class body: - -```python -class _StubFactory: - """Minimal conforming implementation of LogicalTypeFactoryProtocol for use in tests.""" - - def reconstruct_from_arrow(self, arrow_extension_name, storage_type, metadata): - return _StubLogicalType() -``` - -Also update `test_logical_type_factory_create_returns_logical_type` to call `reconstruct_from_arrow`: - -```python -def test_logical_type_factory_create_returns_logical_type(): - """A conforming factory returns a LogicalTypeProtocol from reconstruct_from_arrow.""" - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol, LogicalTypeProtocol - factory: LogicalTypeFactoryProtocol = _StubFactory() - result = factory.reconstruct_from_arrow( - "test.ext", pa.large_utf8(), {"category": "Test"} - ) - assert isinstance(result, LogicalTypeProtocol) -``` - -- [ ] **Step 2: Run the conformance test to confirm it fails (Protocol still expects `create_logical_type`)** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py::test_logical_type_factory_conforming_class_satisfies_protocol -v -``` - -Expected: FAIL — `_StubFactory` is no longer recognized as `LogicalTypeFactoryProtocol` because it lacks `create_logical_type`. - -- [ ] **Step 3: Rename the method in `LogicalTypeFactoryProtocol`** - -In `src/orcapod/extension_types/protocols.py`, rename `create_logical_type` to `reconstruct_from_arrow` in the `LogicalTypeFactoryProtocol` class. The full updated method: - -```python - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict[str, Any], - ) -> LogicalTypeProtocol: - """Reconstruct a LogicalType from Arrow schema metadata (read path). - - Called by the registry when a schema walk encounters an extension type - whose metadata ``"category"`` value matches this factory's registered - category. All Arrow schema information is already known. - - Args: - arrow_extension_name: The Arrow extension type name from the schema. - storage_type: The underlying Arrow storage type. - metadata: Full parsed metadata JSON dict. Always contains ``"category"``. - - Returns: - A fully constructed ``LogicalTypeProtocol`` ready for registration. - - Raises: - ValueError: If this factory cannot reconstruct a type for the given name. - """ - ... -``` - -- [ ] **Step 4: Update the call site in `registry.py`** - -In `src/orcapod/extension_types/registry.py`, find `ensure_extension_type`. Replace: - -```python - logical_type = factory.create_logical_type( - arrow_extension_name, storage_type, metadata_dict - ) -``` - -with: - -```python - logical_type = factory.reconstruct_from_arrow( - arrow_extension_name, storage_type, metadata_dict - ) -``` - -- [ ] **Step 5: Update `_make_stub_factory` in `test_registry.py`** - -In `tests/test_extension_types/test_registry.py`, find `_make_stub_factory`. Replace `create_logical_type` with `reconstruct_from_arrow` in the inner `_Factory` class: - -```python - def reconstruct_from_arrow(self, arrow_extension_name, storage_type, metadata): - self.calls.append((arrow_extension_name, storage_type, metadata)) - if _return_lt is not None: - return _return_lt - return _make_stub(arrow_name=arrow_extension_name, storage=storage_type) -``` - -- [ ] **Step 6: Run the full test suite for extension_types to confirm all pass** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: All previously passing tests still pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py \ - src/orcapod/extension_types/registry.py \ - tests/test_extension_types/test_protocols.py \ - tests/test_extension_types/test_registry.py -git commit -m "refactor(extension_types): rename create_logical_type to reconstruct_from_arrow in LogicalTypeFactoryProtocol" -``` - ---- - -## Task 2: Add `create_for_python_type` to `LogicalTypeFactoryProtocol` - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py` -- Modify: `tests/test_extension_types/test_protocols.py` - -- [ ] **Step 1: Write the failing conformance test** - -Add to `tests/test_extension_types/test_protocols.py`. First update `_StubFactory` to add the new method: - -```python -class _StubFactory: - """Minimal conforming implementation of LogicalTypeFactoryProtocol for use in tests.""" - - def reconstruct_from_arrow(self, arrow_extension_name, storage_type, metadata): - return _StubLogicalType() - - def create_for_python_type(self, python_type): - return _StubLogicalType() -``` - -Then add the test: - -```python -def test_factory_create_for_python_type_conformance(): - """A conforming factory implements create_for_python_type and returns LogicalTypeProtocol.""" - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol, LogicalTypeProtocol - factory: LogicalTypeFactoryProtocol = _StubFactory() - assert isinstance(factory, LogicalTypeFactoryProtocol) - result = factory.create_for_python_type(str) - assert isinstance(result, LogicalTypeProtocol) -``` - -- [ ] **Step 2: Run to confirm it fails (Protocol does not yet require `create_for_python_type`)** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py::test_factory_create_for_python_type_conformance -v -``` - -Expected: FAIL — `LogicalTypeFactoryProtocol` does not yet define `create_for_python_type`, so the `isinstance` check passes but calling an undefined method would fail; or the test passes vacuously — either way, add the method to the Protocol so it becomes structurally required. - -- [ ] **Step 3: Add `create_for_python_type` to `LogicalTypeFactoryProtocol` in `protocols.py`** - -```python - def create_for_python_type( - self, - python_type: type, - ) -> LogicalTypeProtocol: - """Synthesize a LogicalType for the given Python class (write path). - - Called by the registry when pod declaration encounters an unregistered - class whose MRO intersects this factory's registered ``python_bases``. - The factory derives all Arrow metadata (extension name, storage type, - metadata dict) from the Python class itself. - - The returned LogicalType must round-trip: the extension name and metadata - it produces must route back to this same factory's ``reconstruct_from_arrow`` - on a subsequent read. - - Args: - python_type: The concrete Python class to synthesize a LogicalType for. - - Returns: - A fully constructed ``LogicalTypeProtocol`` ready for registration. - - Raises: - ValueError: If this factory cannot construct a type for the given class. - """ - ... -``` - -- [ ] **Step 4: Run to confirm the test passes** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: All pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py \ - tests/test_extension_types/test_protocols.py -git commit -m "feat(extension_types): add create_for_python_type to LogicalTypeFactoryProtocol" -``` - ---- - -## Task 3: Extend `LogicalTypeRegistry` internals and `register_logical_type_factory` - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` -- Modify: `tests/test_extension_types/test_registry.py` - -This task renames `_factories` → `_category_factories`, adds `_python_class_factories`, changes the `register_logical_type_factory` signature, and updates all existing call sites. - -- [ ] **Step 1: Write the new `register_logical_type_factory` tests** - -Add to `tests/test_extension_types/test_registry.py`: - -```python -# ── register_logical_type_factory extended API ─────────────────────────────── - -def test_register_logical_type_factory_keyword_category(): - """register_logical_type_factory accepts factory as first arg, category as keyword.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, category="TestCat") # no error - - -def test_register_logical_type_factory_keyword_python_bases(): - """register_logical_type_factory accepts python_bases as keyword.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[str]) # no error - - -def test_register_logical_type_factory_both_axes(): - """register_logical_type_factory accepts both category and python_bases.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, category="Cat", python_bases=[str, int]) - - -def test_register_logical_type_factory_no_axes_raises(): - """register_logical_type_factory raises ValueError when called with no axes.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - with pytest.raises(ValueError, match="At least one of"): - registry.register_logical_type_factory(factory) - - -def test_register_logical_type_factory_python_base_duplicate_different_factory_raises(): - """Registering a different factory for the same python_base raises ValueError.""" - registry = LogicalTypeRegistry() - f1 = _make_stub_factory() - f2 = _make_stub_factory() - registry.register_logical_type_factory(f1, python_bases=[str]) - with pytest.raises(ValueError): - registry.register_logical_type_factory(f2, python_bases=[str]) - - -def test_register_logical_type_factory_python_base_same_factory_idempotent(): - """Registering the same factory twice for the same python_base is a no-op.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[str]) - registry.register_logical_type_factory(factory, python_bases=[str]) # no error -``` - -- [ ] **Step 2: Run to confirm new tests fail** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -k "keyword_category or keyword_python_bases or both_axes or no_axes or python_base" -v -``` - -Expected: FAIL — `register_logical_type_factory` currently takes `(category, factory)` positionally. - -- [ ] **Step 3: Update existing `register_logical_type_factory` call sites in test_registry.py** - -Search for all existing calls to `register_logical_type_factory` that use the old positional signature `(category, factory)` and update them to the new keyword form `(factory, category=...)`. - -Run this to find them: -```bash -grep -n "register_logical_type_factory" tests/test_extension_types/test_registry.py -``` - -For each occurrence of the form `registry.register_logical_type_factory("SomeCategory", factory)`, replace with `registry.register_logical_type_factory(factory, category="SomeCategory")`. - -- [ ] **Step 4: Update `_make_stub_factory` to also add `create_for_python_type`** - -In `test_registry.py`, update `_make_stub_factory` so the inner `_Factory` class also implements `create_for_python_type` (required by the updated `LogicalTypeFactoryProtocol`): - -```python -def _make_stub_factory(return_lt: LogicalTypeProtocol | None = None) -> LogicalTypeFactoryProtocol: - """Factory for minimal LogicalTypeFactoryProtocol conforming stubs.""" - _return_lt = return_lt - - class _Factory: - def __init__(self): - self.calls: list[tuple] = [] - self.python_type_calls: list[type] = [] - - def reconstruct_from_arrow(self, arrow_extension_name, storage_type, metadata): - self.calls.append((arrow_extension_name, storage_type, metadata)) - if _return_lt is not None: - return _return_lt - return _make_stub(arrow_name=arrow_extension_name, storage=storage_type) - - def create_for_python_type(self, python_type): - self.python_type_calls.append(python_type) - if _return_lt is not None: - return _return_lt - return _make_stub(py_type=python_type) - - return _Factory() -``` - -- [ ] **Step 5: Implement the changes in `registry.py`** - -In `LogicalTypeRegistry.__init__`, rename `_factories` → `_category_factories` and add `_python_class_factories`: - -```python - def __init__(self, logical_types: list[LogicalTypeProtocol] | None = None) -> None: - self._by_logical_name: dict[str, LogicalTypeProtocol] = {} - self._by_arrow_name: dict[str, LogicalTypeProtocol] = {} - self._by_python_type: dict[type, LogicalTypeProtocol] = {} - self._category_factories: dict[str, LogicalTypeFactoryProtocol] = {} - self._python_class_factories: dict[type, LogicalTypeFactoryProtocol] = {} - for lt in (logical_types or []): - self.register_logical_type(lt) -``` - -Replace `register_logical_type_factory` with the new signature. Find the existing method and replace it entirely: - -```python - def register_logical_type_factory( - self, - factory: LogicalTypeFactoryProtocol, - *, - category: str | None = None, - python_bases: Iterable[type] = (), - ) -> None: - """Register a factory on one or both dispatch axes. - - Args: - factory: The factory to register. - category: If given, registers factory as the read-side handler for Arrow - extension types whose metadata contains this category string. Raises - ``ValueError`` if a different factory is already registered for this - category. - python_bases: Zero or more Python base classes. Registers factory as the - write-side handler for each. Raises ``ValueError`` if a different - factory is already registered for a given base. - - Raises: - ValueError: If neither ``category`` nor ``python_bases`` is provided. - ValueError: If a different factory is already registered for a given key. - """ - if category is None and not python_bases: - raise ValueError( - "At least one of 'category' or 'python_bases' must be provided." - ) - if category is not None: - existing = self._category_factories.get(category) - if existing is not None and existing is not factory: - raise ValueError( - f"Cannot register factory for category {category!r}: " - f"a different factory is already registered for this category." - ) - if existing is not factory: - self._category_factories[category] = factory - logger.debug( - "registered LogicalTypeFactory for category %r: %r", category, factory - ) - for base in python_bases: - existing = self._python_class_factories.get(base) - if existing is not None and existing is not factory: - raise ValueError( - f"Cannot register factory for python base {base!r}: " - f"a different factory is already registered for this base." - ) - if existing is not factory: - self._python_class_factories[base] = factory - logger.debug( - "registered LogicalTypeFactory for python base %r: %r", base, factory - ) -``` - -Also update the `ensure_extension_type` method: replace any reference to `self._factories` with `self._category_factories`. - -- [ ] **Step 6: Run all registry tests to confirm they pass** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: All pass. (Any test using the old positional signature was updated in Step 3.) - -- [ ] **Step 7: Run the full extension_types test suite** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: All pass. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/extension_types/registry.py \ - tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add python_class_factories axis to LogicalTypeRegistry; extend register_logical_type_factory" -``` - ---- - -## Task 4: Add `ensure_logical_type_for_python_class` to `LogicalTypeRegistry` - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py` -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Write all failing tests for `ensure_logical_type_for_python_class`** - -Add this block to `tests/test_extension_types/test_registry.py`: - -```python -# ── ensure_logical_type_for_python_class tests ─────────────────────────────── - -class _A: - pass - - -class _B(_A): - pass - - -class _C(_B): - pass - - -def test_ensure_for_python_class_concrete_exact_match(): - """Returns the concrete LogicalType when exact Python type is registered.""" - registry = LogicalTypeRegistry() - lt = _make_stub(py_type=_A) - registry.register_logical_type(lt) - result = registry.ensure_logical_type_for_python_class(_A) - assert result is lt - - -def test_ensure_for_python_class_concrete_mro_match(): - """Returns concrete LogicalType registered for a parent class via MRO walk.""" - registry = LogicalTypeRegistry() - lt = _make_stub(py_type=_A) - registry.register_logical_type(lt) - result = registry.ensure_logical_type_for_python_class(_C) - assert result is lt - - -def test_ensure_for_python_class_factory_synthesis(): - """Calls factory.create_for_python_type and registers the result.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[_A]) - result = registry.ensure_logical_type_for_python_class(_C) - assert len(factory.python_type_calls) == 1 - assert factory.python_type_calls[0] is _C - # Synthesized type is now registered — second call hits cache - cached = registry.ensure_logical_type_for_python_class(_C) - assert cached is result - assert len(factory.python_type_calls) == 1 # factory NOT called again - - -def test_ensure_for_python_class_concrete_beats_factory_same_mro_level(): - """When concrete type and factory are registered for the same class, concrete wins.""" - registry = LogicalTypeRegistry() - lt = _make_stub(py_type=_A) - registry.register_logical_type(lt) - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[_A]) - result = registry.ensure_logical_type_for_python_class(_A) - assert result is lt - assert len(factory.python_type_calls) == 0 # factory never called - - -def test_ensure_for_python_class_factory_more_specific_than_concrete(): - """Factory registered for a subclass beats concrete registered for a parent.""" - registry = LogicalTypeRegistry() - lt_a = _make_stub(py_type=_A) - registry.register_logical_type(lt_a) # concrete for _A - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[_B]) # factory for _B - # Query _C: factory at _B (MRO index 1) beats concrete at _A (MRO index 2) - result = registry.ensure_logical_type_for_python_class(_C) - assert len(factory.python_type_calls) == 1 - assert factory.python_type_calls[0] is _C - - -def test_ensure_for_python_class_concrete_more_specific_than_factory(): - """Concrete registered for a subclass beats factory registered for a parent.""" - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[_A]) # factory for _A - lt_b = _make_stub(py_type=_B) - registry.register_logical_type(lt_b) # concrete for _B - # Query _C: concrete at _B (MRO index 1) beats factory at _A (MRO index 2) - result = registry.ensure_logical_type_for_python_class(_C) - assert result is lt_b - assert len(factory.python_type_calls) == 0 - - -def test_ensure_for_python_class_abc_subclasshook(): - """issubclass fallback scan catches ABCs with __subclasshook__.""" - from abc import ABCMeta - - class _StructuralABC(metaclass=ABCMeta): - @classmethod - def __subclasshook__(cls, C): - return hasattr(C, "_MARKER") - - class _MarkedClass: - _MARKER = True - - registry = LogicalTypeRegistry() - factory = _make_stub_factory() - registry.register_logical_type_factory(factory, python_bases=[_StructuralABC]) - result = registry.ensure_logical_type_for_python_class(_MarkedClass) - assert len(factory.python_type_calls) == 1 - assert factory.python_type_calls[0] is _MarkedClass - - -def test_ensure_for_python_class_no_match_raises_type_error(): - """TypeError raised when no LogicalType and no factory match the type.""" - registry = LogicalTypeRegistry() - - with pytest.raises(TypeError, match="No LogicalType or LogicalTypeFactory"): - registry.ensure_logical_type_for_python_class(_C) -``` - -- [ ] **Step 2: Run to confirm all fail** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -k "ensure_for_python_class" -v -``` - -Expected: All FAIL with `AttributeError: 'LogicalTypeRegistry' has no attribute 'ensure_logical_type_for_python_class'`. - -- [ ] **Step 3: Implement `ensure_logical_type_for_python_class` in `registry.py`** - -Add the method to `LogicalTypeRegistry` after `ensure_extension_type`: - -```python - def ensure_logical_type_for_python_class( - self, - python_type: type, - ) -> LogicalTypeProtocol: - """Ensure a LogicalType exists for python_type, synthesizing via factory if needed. - - Resolution algorithm: - 1. Walk ``python_type.__mro__``. Track the first (most-specific) hit in - ``_by_python_type`` (concrete) and ``_python_class_factories`` (factory) - separately, recording the MRO index of each. - 2. After the MRO walk, if no factory was found, do a fallback ``issubclass`` - scan over ``_python_class_factories`` keys to catch ABCs with - ``__subclasshook__``. Assign these the least-specific MRO index - (len of __mro__) so they lose to any direct MRO match. - 3. Resolution rule: if both concrete and factory found, compare MRO indices — - lower index wins. Ties (same class) → concrete wins. - 4. If factory wins (or only factory found): call - ``factory.create_for_python_type(python_type)``, register the result, - and return it. The registration caches it in ``_by_python_type[python_type]``. - 5. If nothing found: raise ``TypeError``. - - Args: - python_type: The Python class to resolve. - - Returns: - The registered or newly synthesized ``LogicalTypeProtocol``. - - Raises: - TypeError: If no ``LogicalType`` and no factory is found. - """ - best_concrete_idx: int | None = None - best_concrete: LogicalTypeProtocol | None = None - best_factory_idx: int | None = None - best_factory: LogicalTypeFactoryProtocol | None = None - - # Step 1: Walk MRO - for i, base in enumerate(python_type.__mro__): - if best_concrete is None and base in self._by_python_type: - best_concrete_idx = i - best_concrete = self._by_python_type[base] - if best_factory is None and base in self._python_class_factories: - best_factory_idx = i - best_factory = self._python_class_factories[base] - if best_concrete is not None and best_factory is not None: - break - - # Step 2: issubclass fallback scan for ABCs with __subclasshook__ - if best_factory is None: - for base_class, factory in self._python_class_factories.items(): - try: - if issubclass(python_type, base_class): - best_factory = factory - # ABC match — less specific than any direct MRO hit - best_factory_idx = len(python_type.__mro__) - break - except TypeError: - continue - - # Step 3: Resolution - if best_concrete is None and best_factory is None: - raise TypeError( - f"No LogicalType or LogicalTypeFactory is registered for type " - f"{python_type!r}.\n" - f"To handle this type, register a factory for its base class:\n" - f" registry.register_logical_type_factory(\n" - f" factory, python_bases=[]\n" - f" )\n" - f"Or register a concrete LogicalType directly:\n" - f" registry.register_logical_type(my_logical_type)" - ) - - if best_factory is None: - # Only concrete found - assert best_concrete is not None - return best_concrete - - if best_concrete is None: - # Only factory found — synthesize - assert best_factory is not None - lt = best_factory.create_for_python_type(python_type) - self.register_logical_type(lt) - logger.debug( - "ensure_logical_type_for_python_class: synthesized %r for %r", - lt.logical_type_name, - python_type, - ) - return lt - - # Both found — compare specificity (lower MRO index = more specific) - assert best_concrete_idx is not None - assert best_factory_idx is not None - if best_concrete_idx <= best_factory_idx: - # Concrete is same level (ties → concrete wins) or more specific - return best_concrete - else: - # Factory is more specific — synthesize - lt = best_factory.create_for_python_type(python_type) - self.register_logical_type(lt) - logger.debug( - "ensure_logical_type_for_python_class: synthesized %r for %r via more-specific factory", - lt.logical_type_name, - python_type, - ) - return lt -``` - -- [ ] **Step 4: Run the new tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -k "ensure_for_python_class" -v -``` - -Expected: All pass. - -- [ ] **Step 5: Run the full extension_types suite** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: All pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/registry.py \ - tests/test_extension_types/test_registry.py -git commit -m "feat(extension_types): add ensure_logical_type_for_python_class with unified MRO resolution" -``` - ---- - -## Task 5: Add `_extract_leaf_classes` in `type_utils.py` - -**Files:** -- Create: `src/orcapod/extension_types/type_utils.py` -- Modify: `src/orcapod/extension_types/__init__.py` -- Create: `tests/test_extension_types/test_type_utils.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_extension_types/test_type_utils.py`: - -```python -"""Tests for extension_types.type_utils helpers.""" - -from __future__ import annotations - -from typing import Optional, Union - -from orcapod.extension_types.type_utils import _extract_leaf_classes - - -class _A: - pass - - -class _B: - pass - - -def test_plain_class(): - assert list(_extract_leaf_classes(int)) == [int] - - -def test_plain_custom_class(): - assert list(_extract_leaf_classes(_A)) == [_A] - - -def test_list_of_class(): - assert list(_extract_leaf_classes(list[int])) == [int] - - -def test_dict_of_classes(): - result = set(_extract_leaf_classes(dict[str, int])) - assert result == {str, int} - - -def test_optional_unwraps_none(): - """Optional[X] yields X but not NoneType.""" - result = list(_extract_leaf_classes(Optional[int])) - assert result == [int] - - -def test_union_yields_all_non_none(): - result = set(_extract_leaf_classes(Union[int, str])) - assert result == {int, str} - - -def test_union_with_none_excludes_none(): - result = set(_extract_leaf_classes(Union[int, None])) - assert type(None) not in result - assert int in result - - -def test_nested_list_of_dict(): - """list[dict[_A, list[_B]]] yields _A and _B.""" - result = set(_extract_leaf_classes(list[dict[_A, list[_B]]])) - assert result == {_A, _B} - - -def test_deeply_nested(): - """list[dict[str, list[dict[int, _A]]]] yields str, int, _A.""" - result = set(_extract_leaf_classes(list[dict[str, list[dict[int, _A]]]])) - assert result == {str, int, _A} - - -def test_non_generic_non_type_is_skipped(): - """Annotations that are not types and not generic aliases yield nothing.""" - # e.g. a string annotation that failed resolution — should not crash - result = list(_extract_leaf_classes("unresolved_string")) - assert result == [] - - -def test_none_type_plain(): - """type(None) itself yields type(None) as a leaf (not filtered at this level).""" - result = list(_extract_leaf_classes(type(None))) - assert result == [type(None)] -``` - -- [ ] **Step 2: Run to confirm all fail** - -```bash -uv run pytest tests/test_extension_types/test_type_utils.py -v -``` - -Expected: All FAIL with `ModuleNotFoundError` or `ImportError`. - -- [ ] **Step 3: Create `src/orcapod/extension_types/type_utils.py`** - -```python -"""Utility helpers for Python type annotation inspection. - -Used by the write-side registration trigger to extract leaf Python classes from -complex generic annotations like ``list[dict[A, list[B]]]``. -""" - -from __future__ import annotations - -import typing -from typing import Any, Iterator - - -def _extract_leaf_classes(annotation: Any) -> Iterator[type]: - """Recursively yield all concrete leaf Python classes from a type annotation. - - Unwraps generic aliases (``list[T]``, ``dict[K, V]``, ``Optional[T]``, - ``Union[A, B]``, etc.) using ``typing.get_origin`` and ``typing.get_args`` - and yields every non-generic leaf found. ``NoneType`` (from ``Optional`` - and ``Union[..., None]``) is yielded as-is — callers that want to skip it - should filter on ``type(None)``. - - Non-type, non-generic values (e.g. unresolved string annotations) are - silently skipped. - - Args: - annotation: A Python type or generic alias to inspect. - - Yields: - Concrete Python ``type`` objects found at leaf positions. - - Examples: - >>> list(_extract_leaf_classes(list[int])) - [] - >>> set(_extract_leaf_classes(dict[str, list[MyClass]])) - {, } - """ - origin = typing.get_origin(annotation) - - if origin is None: - # Not a generic alias. Yield only if it is a plain type. - if isinstance(annotation, type): - yield annotation - return - - # Generic alias — recurse into every type argument. - for arg in typing.get_args(annotation): - yield from _extract_leaf_classes(arg) -``` - -- [ ] **Step 4: Export from `__init__.py`** - -In `src/orcapod/extension_types/__init__.py`, add to the imports and `__all__`: - -```python -from .type_utils import _extract_leaf_classes -``` - -And add `"_extract_leaf_classes"` to `__all__`. - -- [ ] **Step 5: Run to confirm all tests pass** - -```bash -uv run pytest tests/test_extension_types/test_type_utils.py -v -``` - -Expected: All pass. - -- [ ] **Step 6: Run the full extension_types suite** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: All pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/extension_types/type_utils.py \ - src/orcapod/extension_types/__init__.py \ - tests/test_extension_types/test_type_utils.py -git commit -m "feat(extension_types): add _extract_leaf_classes for recursive generic annotation unwrapping" -``` - ---- - -## Task 6: Wire `LogicalTypeRegistry` into `UniversalTypeConverter` and `DataContext` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Modify: `src/orcapod/contexts/core.py` -- Modify: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Write the failing tests** - -Add to `tests/test_semantic_types/test_universal_converter.py`: - -```python -# ── LogicalTypeRegistry priority tests ─────────────────────────────────────── - -import pyarrow as pa -import polars as pl - -from orcapod.extension_types.registry import ( - LogicalTypeRegistry, - make_arrow_extension_type, - make_polars_extension_type, -) -from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - -def _make_logical_type_stub(py_type: type, arrow_name: str) -> object: - """Return a minimal LogicalTypeProtocol conforming stub.""" - _ArrowExtClass = make_arrow_extension_type(arrow_name, pa.large_string()) - _pl_dtype = pl.String - - class _PolarsExt(pl.BaseExtension): - def __init__(self): - super().__init__(arrow_name, _pl_dtype, None) - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - - class _Stub: - logical_type_name = arrow_name - python_type = py_type - - def get_arrow_extension_type(self): - return _ArrowExtClass() - - def get_polars_extension_type(self): - return _PolarsExt() - - def python_to_storage(self, value): - return str(value) - - def storage_to_python(self, storage_value): - return storage_value - - return _Stub() - - -class _MyCustomClass: - pass - - -def test_converter_uses_logical_type_registry_for_registered_type(): - """When a LogicalType is registered, converter returns its Arrow extension type.""" - import uuid as _uuid - arrow_name = f"test.MyCustomClass.{_uuid.uuid4().hex[:8]}" - lt = _make_logical_type_stub(_MyCustomClass, arrow_name) - - registry = LogicalTypeRegistry() - registry.register_logical_type(lt) - - converter = UniversalTypeConverter() - converter._logical_type_registry = registry - - result = converter.python_type_to_arrow_type(_MyCustomClass) - expected_ext = lt.get_arrow_extension_type() - assert result == expected_ext - - -def test_converter_falls_through_for_unregistered_type(): - """If type not in LogicalTypeRegistry, converter falls through to old system (int → int64).""" - registry = LogicalTypeRegistry() - converter = UniversalTypeConverter() - converter._logical_type_registry = registry - - result = converter.python_type_to_arrow_type(int) - assert result == pa.int64() - - -def test_converter_without_registry_unchanged(): - """With no _logical_type_registry set, converter behaves exactly as before.""" - converter = UniversalTypeConverter() - assert converter.python_type_to_arrow_type(str) == pa.large_string() - - -def test_data_context_wires_registry_into_converter(): - """DataContext.__post_init__ wires logical_type_registry into type_converter.""" - from orcapod.contexts import get_default_context - ctx = get_default_context() - assert hasattr(ctx.type_converter, "_logical_type_registry") - assert ctx.type_converter._logical_type_registry is ctx.logical_type_registry -``` - -- [ ] **Step 2: Run to confirm tests fail** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -k "logical_type_registry or data_context_wires" -v -``` - -Expected: FAIL — `UniversalTypeConverter` has no `_logical_type_registry` attribute. - -- [ ] **Step 3: Add `_logical_type_registry` to `UniversalTypeConverter.__init__`** - -In `src/orcapod/semantic_types/universal_converter.py`, update `__init__`: - -```python - def __init__( - self, - semantic_registry: SemanticTypeRegistry | None = None, - datetime_timezone: typing.Literal["strict", "coerce_utc"] = "strict", - ): - """ - Args: - semantic_registry: Optional registry of semantic type converters. - datetime_timezone: How to handle naive (timezone-less) ``datetime`` - values when converting Python → Arrow. - - ``"strict"`` (default) — raise ``ValueError`` immediately so - callers are forced to be explicit about timezone semantics. - - ``"coerce_utc"`` — silently attach ``timezone.utc`` to naive - datetimes before writing to Arrow. Use this when you know that - all naive datetimes in your data represent UTC. - """ - self.semantic_registry = semantic_registry - self._datetime_timezone = datetime_timezone - self._logical_type_registry = None # set by DataContext.__post_init__ - # ... rest of existing __init__ unchanged ... -``` - -- [ ] **Step 4: Insert the priority check in `_convert_python_to_arrow`** - -In `src/orcapod/semantic_types/universal_converter.py`, find `_convert_python_to_arrow` (around line 411). After the `type_map` check and before the `semantic_registry` check, insert: - -```python - # Check LogicalTypeRegistry first — extension-type identity takes priority - if self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - return lt.get_arrow_extension_type() -``` - -The surrounding context should look like: - -```python - def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: - """Core Python → Arrow type conversion logic.""" - type_map = _get_python_to_arrow_map() - if python_type in type_map: - return type_map[python_type] - - # Check LogicalTypeRegistry first — extension-type identity takes priority - if self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - return lt.get_arrow_extension_type() - - # Check semantic registry for registered types - if self.semantic_registry: - converter = self.semantic_registry.get_converter_for_python_type(python_type) - if converter: - return converter.arrow_struct_type - # ... rest unchanged ... -``` - -- [ ] **Step 5: Add `DataContext.__post_init__` in `contexts/core.py`** - -In `src/orcapod/contexts/core.py`, add a `__post_init__` method to `DataContext`: - -```python - def __post_init__(self) -> None: - """Wire components together after dataclass construction. - - Injects ``logical_type_registry`` into ``type_converter`` so that - registered ``LogicalType`` instances take priority over the old - shape-based ``semantic_registry`` at encoding time. - """ - if hasattr(self.type_converter, "_logical_type_registry"): - self.type_converter._logical_type_registry = self.logical_type_registry -``` - -- [ ] **Step 6: Run the new tests** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -k "logical_type_registry or data_context_wires" -v -``` - -Expected: All pass. - -- [ ] **Step 7: Run the full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -v --tb=short -q -``` - -Expected: All previously passing tests still pass. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py \ - src/orcapod/contexts/core.py \ - tests/test_semantic_types/test_universal_converter.py -git commit -m "feat(extension_types): wire LogicalTypeRegistry into UniversalTypeConverter and DataContext" -``` - ---- - -## Task 7: Add write-side trigger to `_FunctionPodBase` - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` -- Create: `tests/test_core/function_pod/test_write_side_registration.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_core/function_pod/test_write_side_registration.py`: - -```python -"""Tests for write-side LogicalType auto-registration at function pod declaration. - -These tests verify that _FunctionPodBase.__init__ triggers factory synthesis for -any non-native Python types in the pod's input/output schemas, and raises TypeError -at declaration time when no factory is registered. -""" - -from __future__ import annotations - -import dataclasses -import pathlib -import uuid as _uuid_module -from typing import Optional - -import pyarrow as pa -import polars as pl -import pytest - -from orcapod.contexts import get_default_context -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.extension_types.protocols import LogicalTypeProtocol -from orcapod.extension_types.registry import ( - LogicalTypeRegistry, - make_arrow_extension_type, - make_polars_extension_type, -) - - -# ── Helpers ────────────────────────────────────────────────────────────────── - -def _make_registry_with_factory(target_base: type) -> tuple[LogicalTypeRegistry, list]: - """Return a registry with a factory for target_base and a call log.""" - call_log: list[type] = [] - - def _make_lt(py_type: type) -> LogicalTypeProtocol: - arrow_name = f"{py_type.__module__}.{py_type.__qualname__}.{_uuid_module.uuid4().hex[:6]}" - ArrowExt = make_arrow_extension_type(arrow_name, pa.large_string()) - pl_dtype = pl.String - - class _PolarsExt(pl.BaseExtension): - def __init__(self): - super().__init__(arrow_name, pl_dtype, None) - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - - class _LT: - logical_type_name = arrow_name - python_type = py_type - def get_arrow_extension_type(self): return ArrowExt() - def get_polars_extension_type(self): return _PolarsExt() - def python_to_storage(self, v): return str(v) - def storage_to_python(self, v): return v - - return _LT() - - class _Factory: - def reconstruct_from_arrow(self, name, storage, meta): - return _make_lt(object) # unused in these tests - - def create_for_python_type(self, python_type): - call_log.append(python_type) - return _make_lt(python_type) - - registry = LogicalTypeRegistry() - registry.register_logical_type_factory(_Factory(), python_bases=[target_base]) - return registry, call_log - - -# ── Custom classes used in tests ───────────────────────────────────────────── - -class _MyBase: - pass - - -class _MyChild(_MyBase): - pass - - -# ── Tests ──────────────────────────────────────────────────────────────────── - -def test_pod_declaration_triggers_factory_for_unregistered_class(): - """Declaring a FunctionPod with an unregistered type causes factory synthesis.""" - registry, call_log = _make_registry_with_factory(_MyBase) - from orcapod.contexts.core import DataContext - from orcapod.contexts import get_default_context - # Build a context with our test registry - base_ctx = get_default_context() - ctx = DataContext( - context_key="test", - version="test", - description="test", - type_converter=base_ctx.type_converter, - arrow_hasher=base_ctx.arrow_hasher, - semantic_hasher=base_ctx.semantic_hasher, - type_handler_registry=base_ctx.type_handler_registry, - logical_type_registry=registry, - ) - - def my_func(x: _MyChild) -> str: - return str(x) - - # Pod declaration should trigger factory for _MyChild - pod = FunctionPod( - func=my_func, - output_keys=["result"], - data_context=ctx, - ) - assert _MyChild in call_log - # The synthesized LogicalType is now in the registry - assert registry.get_by_python_type(_MyChild) is not None - - -def test_pod_declaration_with_nested_list_type(): - """list[_MyChild] in the schema causes factory synthesis for _MyChild.""" - registry, call_log = _make_registry_with_factory(_MyBase) - from orcapod.contexts.core import DataContext - from orcapod.contexts import get_default_context - base_ctx = get_default_context() - ctx = DataContext( - context_key="test", - version="test", - description="test", - type_converter=base_ctx.type_converter, - arrow_hasher=base_ctx.arrow_hasher, - semantic_hasher=base_ctx.semantic_hasher, - type_handler_registry=base_ctx.type_handler_registry, - logical_type_registry=registry, - ) - - def my_func(items: list[_MyChild]) -> str: - return "" - - FunctionPod(func=my_func, output_keys=["result"], data_context=ctx) - assert _MyChild in call_log - - -def test_pod_declaration_native_types_no_factory_call(): - """Pods using only native types (int, str, etc.) never trigger factory lookup.""" - registry = LogicalTypeRegistry() - - class _NeverCalledFactory: - def reconstruct_from_arrow(self, *a): ... - def create_for_python_type(self, pt): - raise AssertionError(f"factory called for {pt!r}") - - registry.register_logical_type_factory( - _NeverCalledFactory(), python_bases=[object] - ) - from orcapod.contexts.core import DataContext - from orcapod.contexts import get_default_context - base_ctx = get_default_context() - ctx = DataContext( - context_key="test", version="test", description="test", - type_converter=base_ctx.type_converter, - arrow_hasher=base_ctx.arrow_hasher, - semantic_hasher=base_ctx.semantic_hasher, - type_handler_registry=base_ctx.type_handler_registry, - logical_type_registry=registry, - ) - - def my_func(x: int, y: str) -> float: - return 0.0 - - # Should not raise — int, str, float are native - FunctionPod(func=my_func, output_keys=["result"], data_context=ctx) - - -def test_pod_declaration_raises_type_error_for_unhandled_class(): - """Pod with a type that has no registered factory raises TypeError at declaration.""" - registry = LogicalTypeRegistry() # empty — no factories - from orcapod.contexts.core import DataContext - from orcapod.contexts import get_default_context - base_ctx = get_default_context() - ctx = DataContext( - context_key="test", version="test", description="test", - type_converter=base_ctx.type_converter, - arrow_hasher=base_ctx.arrow_hasher, - semantic_hasher=base_ctx.semantic_hasher, - type_handler_registry=base_ctx.type_handler_registry, - logical_type_registry=registry, - ) - - def my_func(x: _MyChild) -> str: - return "" - - with pytest.raises(TypeError, match="No LogicalType or LogicalTypeFactory"): - FunctionPod(func=my_func, output_keys=["result"], data_context=ctx) - - -def test_pod_declaration_already_registered_type_no_factory_call(): - """Pre-registered types are not passed to the factory.""" - registry, call_log = _make_registry_with_factory(_MyBase) - # Pre-register _MyChild directly - from orcapod.extension_types.registry import make_arrow_extension_type - ArrowExt = make_arrow_extension_type(f"test.MyChild.{_uuid_module.uuid4().hex[:6]}", pa.large_string()) - - class _PreLT: - logical_type_name = f"test.{_uuid_module.uuid4().hex[:6]}" - python_type = _MyChild - def get_arrow_extension_type(self): return ArrowExt() - def get_polars_extension_type(self): - class P(pl.BaseExtension): - def __init__(self): super().__init__(self.logical_type_name, pl.String, None) - @classmethod - def ext_from_params(cls, *a): return cls() - return P() - def python_to_storage(self, v): return str(v) - def storage_to_python(self, v): return v - - registry.register_logical_type(_PreLT()) - from orcapod.contexts.core import DataContext - from orcapod.contexts import get_default_context - base_ctx = get_default_context() - ctx = DataContext( - context_key="test", version="test", description="test", - type_converter=base_ctx.type_converter, - arrow_hasher=base_ctx.arrow_hasher, - semantic_hasher=base_ctx.semantic_hasher, - type_handler_registry=base_ctx.type_handler_registry, - logical_type_registry=registry, - ) - - def my_func(x: _MyChild) -> str: - return "" - - FunctionPod(func=my_func, output_keys=["result"], data_context=ctx) - # Factory was NOT called — _MyChild was already registered - assert _MyChild not in call_log -``` - -- [ ] **Step 2: Run to confirm all fail** - -```bash -uv run pytest tests/test_core/function_pod/test_write_side_registration.py -v -``` - -Expected: All FAIL — the trigger does not exist yet. - -- [ ] **Step 3: Implement the trigger in `function_pod.py`** - -Add imports at the top of `src/orcapod/core/function_pod.py` (with the existing imports): - -```python -from orcapod.extension_types.type_utils import _extract_leaf_classes -from orcapod.extension_types.registry import LogicalTypeRegistry -``` - -Add the module-level constant and helper function before the `_FunctionPodBase` class definition: - -```python -# Python types that Arrow handles natively — no LogicalType registration needed. -_ARROW_NATIVE_TYPES: frozenset[type] = frozenset({ - int, float, str, bytes, bool, type(None), -}) - - -def _trigger_write_side_registration( - input_schema: Schema, - output_schema: Schema, - registry: LogicalTypeRegistry | None, -) -> None: - """Ensure a LogicalType is registered for every non-native leaf class in the schemas. - - Called once at pod declaration time. Recursively unwraps generic annotations - (``list[T]``, ``dict[K, V]``, etc.) to find leaf classes. Skips Arrow-native - types and already-registered types. Raises ``TypeError`` at declaration time - if no factory is registered for a leaf class — this is intentional. - - Args: - input_schema: The pod's input data schema (column name → Python type annotation). - output_schema: The pod's output data schema. - registry: The ``LogicalTypeRegistry`` from the pod's ``DataContext``. - If ``None``, this function is a no-op. - """ - if registry is None: - return - for schema in (input_schema, output_schema): - for annotation in schema.values(): - for leaf_class in _extract_leaf_classes(annotation): - if leaf_class in _ARROW_NATIVE_TYPES: - continue - if registry.get_by_python_type(leaf_class) is not None: - continue # already registered — O(1) cache hit - registry.ensure_logical_type_for_python_class(leaf_class) - # TypeError propagates if no factory matches — intentional hard error -``` - -In `_FunctionPodBase.__init__`, add the trigger call after `self._data_function = data_function`: - -```python - self._data_function = data_function - _trigger_write_side_registration( - data_function.input_data_schema, - data_function.output_data_schema, - self.data_context.logical_type_registry, - ) -``` - -- [ ] **Step 4: Run the new tests** - -```bash -uv run pytest tests/test_core/function_pod/test_write_side_registration.py -v -``` - -Expected: All pass. - -- [ ] **Step 5: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short -q -``` - -Expected: All previously passing tests still pass. The trigger is a no-op for native types and already-registered built-ins (Path, UPath, UUID), so existing pod tests are unaffected. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/function_pod.py \ - tests/test_core/function_pod/test_write_side_registration.py -git commit -m "feat(extension_types): add write-side registration trigger in _FunctionPodBase.__init__" -``` - ---- - -## Self-Review Checklist - -**Spec coverage:** - -| Spec section | Covered by task | -|---|---| -| `reconstruct_from_arrow` rename | Task 1 | -| `create_for_python_type` new method | Task 2 | -| `_category_factories` rename, `_python_class_factories`, extended `register_logical_type_factory` | Task 3 | -| `ensure_logical_type_for_python_class` with unified MRO resolution, caching, TypeError | Task 4 | -| `_extract_leaf_classes` for complex nested annotations | Task 5 | -| `UniversalTypeConverter` priority check + `DataContext` wiring | Task 6 | -| `_trigger_write_side_registration`, `_ARROW_NATIVE_TYPES`, `_FunctionPodBase.__init__` call | Task 7 | -| Failure mode: hard TypeError at declaration time | Task 7 tests | -| Symmetry with read side (protocol contract documented) | Task 2 docstring | -| Built-in types unaffected | Task 7 tests (native types test, pre-registered test) | - -**Type consistency across tasks:** -- `reconstruct_from_arrow` defined in Task 1, used in Task 3 (factory stub) — consistent ✓ -- `create_for_python_type` defined in Task 2, tested in Task 4 (`python_type_calls`) — consistent ✓ -- `_category_factories` introduced in Task 3, referenced in `ensure_logical_type_for_python_class` Task 4 — consistent ✓ -- `_python_class_factories` introduced in Task 3, used in Task 4 — consistent ✓ -- `_extract_leaf_classes` created in Task 5, imported in Task 7 — consistent ✓ -- `_logical_type_registry` attribute name defined in Task 6, checked in Task 6's DataContext test — consistent ✓ -- `LogicalTypeRegistry` import added to `function_pod.py` in Task 7 type annotation — consistent ✓ diff --git a/superpowers/plans/2026-06-16-plt-1705-type-registration-spine-refactor.md b/superpowers/plans/2026-06-16-plt-1705-type-registration-spine-refactor.md deleted file mode 100644 index 1505b8bab..000000000 --- a/superpowers/plans/2026-06-16-plt-1705-type-registration-spine-refactor.md +++ /dev/null @@ -1,2897 +0,0 @@ -# PLT-1705 Type Registration Spine Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `UniversalTypeConverter` the single re-entry point for Python ↔ Arrow type registration, move `LogicalTypeRegistry` inside the converter as a private implementation detail, and implement `DataclassHandlerFactory` on the refined architecture. - -**Architecture:** `register_python_class(annotation)` handles write-side recursive traversal; `register_storage_type(arrow_type)` handles read-side bottom-up traversal. Factories and logical types receive `converter` instead of `registry`, so all delegation flows through the converter. `DataContext.logical_type_registry` is removed entirely. - -**Tech Stack:** Python 3.12, PyArrow, Polars, `dataclasses`, `typing.get_type_hints` - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/extension_types/protocols.py` | Modify | Add `TypeConverterProtocol`; add `supports_class` + `converter` param to factory protocol; add `converter` param to logical type protocol | -| `src/orcapod/extension_types/builtin_logical_types.py` | Modify | Add `converter` param (accept, ignore) to `python_to_storage` / `storage_to_python` | -| `src/orcapod/semantic_types/universal_converter.py` | Modify | Add `register_python_class`, `register_storage_type`, `python_to_storage`, `storage_to_python`, `register_logical_type`, `register_logical_type_factory`; update `_create_python_to_arrow_converter`/`_create_arrow_to_python_converter` to pass `converter=self`; simplify `ensure_types_registered_for_schemas`; remove `semantic_registry` usage; remove `dataclass_encoding` imports | -| `src/orcapod/extension_types/registry.py` | Modify | Remove `ensure_logical_type_for_python_class`, `ensure_extension_type` | -| `src/orcapod/extension_types/dataclass_handler.py` | **Create** | `DataclassLogicalType` + `DataclassHandlerFactory` | -| `src/orcapod/semantic_types/dataclass_encoding.py` | **Delete** | Superseded by `DataclassHandlerFactory` | -| `src/orcapod/extension_types/type_utils.py` | Modify | Rename `extract_leaf_classes` → `_extract_leaf_classes` (private) | -| `src/orcapod/extension_types/database_hooks.py` | Modify | `register_discovered_extensions` takes `converter` instead of `registry`; uses schema_walker + `converter._ensure_extension_type_info` | -| `src/orcapod/databases/extension_aware_database.py` | Modify | Takes `converter` instead of `registry`; passes `converter._registry` to `apply_extension_types` | -| `src/orcapod/contexts/core.py` | Modify | Remove `logical_type_registry` field from `DataContext` | -| `src/orcapod/contexts/__init__.py` | Modify | Remove `get_default_logical_type_registry` | -| `src/orcapod/contexts/registry.py` | Modify | Remove `"logical_type_registry"` from `required_fields`; stop passing it to `DataContext` | -| `src/orcapod/contexts/data/v0.1.json` | Modify | Remove top-level `logical_type_registry`; move registry construction inside `type_converter._config`; remove `semantic_registry` ref from `type_converter._config` | -| `src/orcapod/contexts/data/schemas/context_schema.json` | Modify | Remove `logical_type_registry` from `required` and `properties` | -| `src/orcapod/extension_types/__init__.py` | Modify | Update docstring | -| `tests/test_extension_types/test_protocols.py` | Modify | Update stubs for new signatures; add `TypeConverterProtocol` conformance test | -| `tests/test_extension_types/test_registry.py` | Modify | Remove `ensure_*` tests; add converter pass-through tests | -| `tests/test_extension_types/test_builtin_logical_types.py` | Modify | Pass a stub converter to `python_to_storage` / `storage_to_python` calls | -| `tests/test_extension_types/test_dataclass_handler.py` | **Create** | Full unit tests for `DataclassLogicalType` and `DataclassHandlerFactory` | -| `tests/test_semantic_types/test_universal_converter.py` | Modify | Add `register_python_class` and `register_storage_type` tests | -| `tests/test_extension_types/test_database_hooks.py` | Modify | Switch from registry to converter | -| `tests/test_core/function_pod/test_write_side_registration.py` | Modify | Update `DataContext` construction (no `logical_type_registry`) | - ---- - -## Task 1: Update `TypeConverterProtocol` and factory/logical-type protocols - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py` -- Modify: `tests/test_extension_types/test_protocols.py` - -- [ ] **Step 1: Write failing protocol conformance tests** - -Add to `tests/test_extension_types/test_protocols.py`: - -```python -# Add at the top of the file: -# from orcapod.extension_types.protocols import TypeConverterProtocol - -def test_type_converter_protocol_is_importable(): - from orcapod.extension_types.protocols import TypeConverterProtocol - assert TypeConverterProtocol is not None - - -def test_factory_supports_class_method_required(): - """LogicalTypeFactoryProtocol requires supports_class.""" - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol - - class _BadFactory: - def reconstruct_from_arrow(self, name, storage_type, metadata, converter): - pass - def create_for_python_type(self, python_type, converter): - pass - # Missing supports_class - - assert not isinstance(_BadFactory(), LogicalTypeFactoryProtocol) - - -def test_factory_with_supports_class_satisfies_protocol(): - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol - - class _GoodFactory: - def supports_class(self, python_type): - return True - def reconstruct_from_arrow(self, name, storage_type, metadata, converter): - pass - def create_for_python_type(self, python_type, converter): - pass - - assert isinstance(_GoodFactory(), LogicalTypeFactoryProtocol) - - -def test_logical_type_python_to_storage_accepts_converter(): - """LogicalTypeProtocol.python_to_storage now requires converter param.""" - from orcapod.extension_types.protocols import LogicalTypeProtocol - - class _GoodLT: - @property - def logical_type_name(self): return "test.lt" - @property - def python_type(self): return str - def get_arrow_extension_type(self): pass - def get_polars_extension_type(self): pass - def python_to_storage(self, value, converter): return value - def storage_to_python(self, storage_value, converter): return storage_value - - assert isinstance(_GoodLT(), LogicalTypeProtocol) -``` - -- [ ] **Step 2: Run tests to confirm failures** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -k "type_converter or supports_class or accepts_converter" 2>&1 | tail -30 -``` -Expected: ImportError or AttributeError failures. - -- [ ] **Step 3: Update `protocols.py`** - -Replace the entire file: - -```python -"""Protocol definitions for the Arrow/Polars extension type system. - -This module defines ``TypeConverterProtocol``, ``LogicalTypeProtocol``, and -``LogicalTypeFactoryProtocol`` — the contracts for the converter, for logical -type implementations that bind a Python class to its Arrow and Polars extension -type representation, and for factories that auto-construct such implementations -from Arrow schema metadata. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - - -@runtime_checkable -class TypeConverterProtocol(Protocol): - """Minimal protocol exposing what factories and logical types need from the converter. - - Placed in ``extension_types/protocols.py`` to avoid circular imports. - ``UniversalTypeConverter`` is the canonical implementation. - """ - - def register_python_class(self, annotation: Any) -> "pa.DataType": - """Traverse a Python annotation and return its Arrow type, registering as needed.""" - ... - - def register_storage_type(self, arrow_type: "pa.DataType") -> "pa.DataType": - """Traverse an Arrow type bottom-up, registering extension types, and return resolved type.""" - ... - - def python_to_storage(self, value: Any, annotation: Any) -> Any: - """Convert a Python value to its Arrow storage representation.""" - ... - - def storage_to_python(self, storage_value: Any, annotation: Any) -> Any: - """Convert an Arrow storage value back to a Python object.""" - ... - - -@runtime_checkable -class LogicalTypeProtocol(Protocol): - """Protocol for Arrow/Polars extension-type-backed logical types. - - A ``LogicalTypeProtocol`` is a three-way binding between a unique logical type name - (orcapod's identifier), a Python class, and Arrow/Polars extension types. - Each implementation *owns* its Arrow and Polars extension types by providing - them directly via ``get_arrow_extension_type`` and ``get_polars_extension_type``. - - This protocol is Arrow I/O only — hashing is not a logical type responsibility. - """ - - @property - def logical_type_name(self) -> str: - """Unique orcapod identifier for this logical type (e.g. ``"orcapod.uuid"``).""" - ... - - @property - def python_type(self) -> type: - """The Python class this logical type represents.""" - ... - - def get_arrow_extension_type(self) -> "pa.ExtensionType": - """Return the Arrow extension type for this logical type.""" - ... - - def get_polars_extension_type(self) -> "pl.BaseExtension": - """Return an instance of the Polars extension type for this logical type.""" - ... - - def python_to_storage(self, value: Any, converter: TypeConverterProtocol) -> Any: - """Convert a Python value to its Arrow storage representation. - - Args: - value: A Python object of type ``python_type``. - converter: The active ``TypeConverterProtocol`` for recursive delegation. - - Returns: - A value suitable for Arrow storage. - """ - ... - - def storage_to_python(self, storage_value: Any, converter: TypeConverterProtocol) -> Any: - """Convert an Arrow storage value back to a Python object. - - Args: - storage_value: A scalar or array element from the Arrow storage array. - converter: The active ``TypeConverterProtocol`` for recursive delegation. - - Returns: - A Python object of type ``python_type``. - """ - ... - - -@runtime_checkable -class LogicalTypeFactoryProtocol(Protocol): - """Protocol for factories that synthesize or reconstruct ``LogicalTypeProtocol`` instances. - - Bridges two directions: the write path (``create_for_python_type``) and the read - path (``reconstruct_from_arrow``). Both methods receive ``converter`` instead of - ``registry`` so all traversal flows through the converter. - """ - - def supports_class(self, python_type: type) -> bool: - """Return True if this factory can synthesize a LogicalType for ``python_type``. - - Used as a probe during write-side MRO dispatch in ``register_python_class``. - - Args: - python_type: The Python class to probe. - - Returns: - True if this factory handles ``python_type``. - """ - ... - - def create_for_python_type( - self, - python_type: type, - converter: TypeConverterProtocol, - ) -> LogicalTypeProtocol: - """Synthesize a LogicalType for the given Python class (write path). - - Args: - python_type: The concrete Python class to synthesize a LogicalType for. - converter: The active converter for recursive field-type resolution. - - Returns: - A fully constructed ``LogicalTypeProtocol`` ready for registration. - - Raises: - ValueError: If this factory cannot construct a type for the given class. - """ - ... - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: "pa.DataType", - metadata: dict[str, Any], - converter: TypeConverterProtocol, - ) -> LogicalTypeProtocol: - """Reconstruct a LogicalType from Arrow schema metadata (read path). - - Args: - arrow_extension_name: The Arrow extension type name from the schema. - storage_type: The underlying Arrow storage type (already resolved bottom-up). - metadata: Full parsed metadata JSON dict. Always contains ``"category"``. - converter: The active converter for recursive field-type resolution. - - Returns: - A fully constructed ``LogicalTypeProtocol`` ready for registration. - - Raises: - ValueError: If this factory cannot reconstruct a type for the given name. - """ - ... -``` - -- [ ] **Step 4: Run tests to confirm they pass** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v 2>&1 | tail -30 -``` -Expected: All tests pass (some existing tests about the OLD signatures will now fail — that's expected and will be fixed in Task 2). - -- [ ] **Step 5: Update existing stubs in `test_protocols.py` to use new signatures** - -Replace `_StubLogicalType` and `_StubFactory` in `tests/test_extension_types/test_protocols.py`: - -```python -class _StubLogicalType: - """Minimal conforming implementation of LogicalTypeProtocol for use in tests.""" - - _ArrowExtClass = make_arrow_extension_type("test.module.MyType", pa.large_string()) - - @property - def logical_type_name(self) -> str: - return "test.module.MyType" - - @property - def python_type(self) -> type: - return str - - def get_arrow_extension_type(self) -> pa.ExtensionType: - return self._ArrowExtClass() - - def get_polars_extension_type(self) -> pl.BaseExtension: - class _PolarsExt(pl.BaseExtension): - def __init__(self): - super().__init__("test.module.MyType", pl.String, None) - @classmethod - def ext_from_params(cls, ext_name, storage_dtype, metadata_str): - return cls() - return _PolarsExt() - - def python_to_storage(self, value, converter): # converter param added - return str(value) - - def storage_to_python(self, storage_value, converter): # converter param added - return storage_value - - -class _StubFactory: - """Minimal conforming implementation of LogicalTypeFactoryProtocol for use in tests.""" - - def supports_class(self, python_type): # new method - return True - - def reconstruct_from_arrow(self, arrow_extension_name, storage_type, metadata, converter): - return _StubLogicalType() - - def create_for_python_type(self, python_type, converter): - return _StubLogicalType() -``` - -Also update the test that calls the old signatures: -```python -def test_conforming_class_satisfies_protocol(): - lt: LogicalTypeProtocol = _StubLogicalType() - assert lt.logical_type_name == "test.module.MyType" - assert lt.python_type is str - assert lt.get_arrow_extension_type().extension_name == "test.module.MyType" - assert isinstance(lt.get_polars_extension_type(), pl.BaseExtension) - assert lt.python_to_storage(42, None) == "42" # pass converter=None - assert lt.storage_to_python("hello", None) == "hello" # pass converter=None - - -def test_logical_type_factory_create_returns_logical_type(): - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol, LogicalTypeProtocol - factory: LogicalTypeFactoryProtocol = _StubFactory() - result = factory.reconstruct_from_arrow( - "test.ext", pa.large_utf8(), {"category": "Test"}, converter=None - ) - assert isinstance(result, LogicalTypeProtocol) - - -def test_factory_create_for_python_type_conformance(): - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol, LogicalTypeProtocol - factory: LogicalTypeFactoryProtocol = _StubFactory() - assert isinstance(factory, LogicalTypeFactoryProtocol) - result = factory.create_for_python_type(str, converter=None) - assert isinstance(result, LogicalTypeProtocol) -``` - -- [ ] **Step 6: Run all protocol tests** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v 2>&1 | tail -20 -``` -Expected: All pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py tests/test_extension_types/test_protocols.py -git commit -m "feat(extension_types): add TypeConverterProtocol; update factory/logical-type protocols with converter param and supports_class" -``` - ---- - -## Task 2: Update built-in logical types for protocol conformance - -**Files:** -- Modify: `src/orcapod/extension_types/builtin_logical_types.py` -- Modify: `tests/test_extension_types/test_builtin_logical_types.py` - -- [ ] **Step 1: Write failing tests** - -Add to `tests/test_extension_types/test_builtin_logical_types.py`: - -```python -def test_logical_path_python_to_storage_accepts_converter(): - """python_to_storage now accepts a converter param (ignored).""" - from orcapod.extension_types.builtin_logical_types import LogicalPath - lt = LogicalPath() - import pathlib - result = lt.python_to_storage(pathlib.Path("/tmp/foo"), converter=None) - assert result == "/tmp/foo" - - -def test_logical_uuid_python_to_storage_accepts_converter(): - from orcapod.extension_types.builtin_logical_types import LogicalUUID - import uuid as uuid_module - lt = LogicalUUID() - u = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - result = lt.python_to_storage(u, converter=None) - assert result == u.bytes - - -def test_logical_upath_storage_to_python_accepts_converter(): - from orcapod.extension_types.builtin_logical_types import LogicalUPath - lt = LogicalUPath() - from upath import UPath - result = lt.storage_to_python("s3://bucket/key", converter=None) - assert isinstance(result, UPath) -``` - -- [ ] **Step 2: Run to confirm failures** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v -k "accepts_converter" 2>&1 | tail -20 -``` -Expected: TypeError — unexpected keyword argument. - -- [ ] **Step 3: Update all three classes in `builtin_logical_types.py`** - -For `LogicalPath`: -```python -def python_to_storage(self, value: Any, converter: Any = None) -> str: - return str(value) - -def storage_to_python(self, storage_value: Any, converter: Any = None) -> pathlib.Path: - return pathlib.Path(storage_value) -``` - -For `LogicalUPath`: -```python -def python_to_storage(self, value: Any, converter: Any = None) -> str: - return str(value) - -def storage_to_python(self, storage_value: Any, converter: Any = None) -> UPath: - return UPath(storage_value) -``` - -For `LogicalUUID`: -```python -def python_to_storage(self, value: Any, converter: Any = None) -> bytes: - return value.bytes - -def storage_to_python(self, storage_value: Any, converter: Any = None) -> _uuid_module.UUID: - return _uuid_module.UUID(bytes=bytes(storage_value)) -``` - -Also add `TYPE_CHECKING` import for `TypeConverterProtocol` in the type hint: -```python -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from orcapod.extension_types.protocols import TypeConverterProtocol -``` - -And use in signatures: -```python -def python_to_storage(self, value: Any, converter: "TypeConverterProtocol | None" = None) -> str: -``` - -- [ ] **Step 4: Run new tests** - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v 2>&1 | tail -20 -``` -Expected: All pass. - -- [ ] **Step 5: Also update test call sites that call without converter** - -Search for existing direct calls to `python_to_storage` / `storage_to_python` in the test file (they have no `converter` arg — that's fine since we added `converter=None` default). - -```bash -uv run pytest tests/test_extension_types/test_builtin_logical_types.py -v 2>&1 | tail -5 -``` -Expected: All pass (defaults handle existing calls). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/builtin_logical_types.py tests/test_extension_types/test_builtin_logical_types.py -git commit -m "feat(extension_types): add converter param to built-in logical type python_to_storage/storage_to_python" -``` - ---- - -## Task 3: Add `register_python_class` to `UniversalTypeConverter` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Modify: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Write failing tests** - -Add to `tests/test_semantic_types/test_universal_converter.py`: - -```python -import dataclasses -import uuid as _uuid_module -import pathlib -from typing import Optional - -import pyarrow as pa -import pytest - -from orcapod.extension_types.registry import LogicalTypeRegistry, make_arrow_extension_type, make_polars_extension_type -from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - -# ── Helpers ───────────────────────────────────────────────────────────────── - -def _make_registry_with_builtins() -> LogicalTypeRegistry: - """Registry with LogicalPath, LogicalUUID, LogicalUPath pre-registered.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath, LogicalUUID, LogicalUPath - return LogicalTypeRegistry(logical_types=[LogicalPath(), LogicalUUID(), LogicalUPath()]) - - -def _make_converter(registry: LogicalTypeRegistry | None = None) -> UniversalTypeConverter: - if registry is None: - registry = _make_registry_with_builtins() - return UniversalTypeConverter(logical_type_registry=registry) - - -# ── register_python_class tests ────────────────────────────────────────────── - -def test_register_python_class_primitive_int(): - converter = _make_converter() - assert converter.register_python_class(int) == pa.int64() - - -def test_register_python_class_primitive_str(): - converter = _make_converter() - assert converter.register_python_class(str) == pa.large_string() - - -def test_register_python_class_list_of_int(): - converter = _make_converter() - result = converter.register_python_class(list[int]) - assert result == pa.large_list(pa.int64()) - - -def test_register_python_class_optional_str(): - converter = _make_converter() - result = converter.register_python_class(Optional[str]) - assert result == pa.large_string() - - -def test_register_python_class_dict_str_int(): - converter = _make_converter() - result = converter.register_python_class(dict[str, int]) - expected = pa.large_list(pa.struct([pa.field("key", pa.large_string()), pa.field("value", pa.int64())])) - assert result == expected - - -def test_register_python_class_set_of_str(): - converter = _make_converter() - result = converter.register_python_class(set[str]) - assert result == pa.large_list(pa.large_string()) - - -def test_register_python_class_registry_hit_path(): - """pathlib.Path is pre-registered → returns the orcapod.path extension type.""" - converter = _make_converter() - result = converter.register_python_class(pathlib.Path) - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "orcapod.path" - - -def test_register_python_class_uuid_registry_hit(): - converter = _make_converter() - result = converter.register_python_class(_uuid_module.UUID) - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "orcapod.uuid" - - -def test_register_python_class_factory_dispatch(): - """A custom class triggers factory synthesis and caches the result.""" - import uuid as _u - import polars as pl - - class _Base: - pass - - class _Child(_Base): - pass - - ext_name = f"test.custom.{_u.uuid4().hex[:8]}" - ArrowExt = make_arrow_extension_type(ext_name, pa.large_string()) - PolarsExt = make_polars_extension_type(ext_name, pa.large_string()) - synthesized_calls = [] - - class _Factory: - def supports_class(self, python_type): - return issubclass(python_type, _Base) - def create_for_python_type(self, python_type, converter): - synthesized_calls.append(python_type) - class _LT: - logical_type_name = ext_name - python_type_ = _Child - python_type = _Child - def get_arrow_extension_type(self): return ArrowExt() - def get_polars_extension_type(self): return PolarsExt() - def python_to_storage(self, v, c=None): return str(v) - def storage_to_python(self, v, c=None): return v - return _LT() - def reconstruct_from_arrow(self, name, storage, meta, converter): pass - - registry = _make_registry_with_builtins() - registry.register_logical_type_factory(_Factory(), python_bases=[_Base]) - converter = _make_converter(registry) - - result = converter.register_python_class(_Child) - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == ext_name - assert _Child in synthesized_calls - - # Second call is a registry hit — factory NOT called again - result2 = converter.register_python_class(_Child) - assert result2 == result - assert len(synthesized_calls) == 1 - - -def test_register_python_class_cycle_detection(): - """Cyclic type synthesis raises TypeError.""" - import uuid as _u - import polars as pl - - class _CycleClass: - pass - - class _CycleFactory: - def supports_class(self, python_type): - return python_type is _CycleClass - def create_for_python_type(self, python_type, converter): - # Intentionally trigger a cycle - converter.register_python_class(_CycleClass) - def reconstruct_from_arrow(self, name, storage, meta, converter): pass - - registry = _make_registry_with_builtins() - registry.register_logical_type_factory(_CycleFactory(), python_bases=[_CycleClass]) - converter = _make_converter(registry) - - with pytest.raises(TypeError, match="[Cc]ircular"): - converter.register_python_class(_CycleClass) -``` - -- [ ] **Step 2: Run to confirm failures** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -k "register_python_class" 2>&1 | tail -30 -``` -Expected: AttributeError — `UniversalTypeConverter` has no attribute `register_python_class`. - -- [ ] **Step 3: Implement `register_python_class` in `UniversalTypeConverter`** - -Add these methods to `UniversalTypeConverter` (after `__init__`): - -```python -def register_python_class(self, annotation: Any) -> "pa.DataType": - """Register a Python type annotation and return its Arrow type. - - Traverses generic annotations recursively. For each concrete class found, - either returns from the primitive map or registry (cache hit), or - synthesises via factory and registers the result. - - Args: - annotation: A Python type or generic alias (e.g. ``list[str]``, - ``Optional[uuid.UUID]``, a dataclass type). - - Returns: - The Arrow ``pa.DataType`` corresponding to ``annotation``. - - Raises: - TypeError: If a concrete class has no registered ``LogicalType`` and - no factory covers it, or if a circular dependency is detected. - ValueError: If a complex (non-Optional) union is encountered. - """ - import types as _types_mod - - type_map = _get_python_to_arrow_map() - - # Primitive map hit - if annotation in type_map: - return type_map[annotation] - - origin = get_origin(annotation) - args = get_args(annotation) - - # Optional[T] / T | None → strip None arm - if origin is typing.Union or origin is _types_mod.UnionType: - non_none = [a for a in args if a is not type(None)] - if len(non_none) == 1: - return self.register_python_class(non_none[0]) - raise ValueError( - f"Complex unions with multiple non-None types are not supported: " - f"{annotation!r}. Only Optional[T] (T | None) is allowed." - ) - - # list[T] → pa.large_list(T) - if origin is list: - return pa.large_list(self.register_python_class(args[0])) - - # set[T] → pa.large_list(T) - if origin is set: - return pa.large_list(self.register_python_class(args[0])) - - # dict[K, V] → pa.large_list(struct{key: K, value: V}) - if origin is dict: - key_arrow = self.register_python_class(args[0]) - val_arrow = self.register_python_class(args[1]) - return pa.large_list( - pa.struct([pa.field("key", key_arrow), pa.field("value", val_arrow)]) - ) - - # Concrete class — registry or factory dispatch - if isinstance(annotation, type): - if self._logical_type_registry is None: - raise TypeError( - f"No LogicalTypeRegistry configured — cannot register {annotation!r}. " - f"Provide logical_type_registry at converter construction time." - ) - - # Registry hit (already synthesised) - lt = self._logical_type_registry.get_by_python_type(annotation) - if lt is not None: - return lt.get_arrow_extension_type() - - # Cycle detection - if annotation in self._in_progress: - raise TypeError( - f"Circular type dependency detected while synthesising " - f"LogicalType for {annotation!r}." - ) - - # Factory dispatch via MRO walk - factory = self._find_factory_for_class(annotation) - if factory is None: - raise TypeError( - f"No LogicalType or LogicalTypeFactory registered for {annotation!r}. " - f"Register a factory: converter.register_logical_type_factory(factory, " - f"python_bases=[])" - ) - - self._in_progress.add(annotation) - try: - lt = factory.create_for_python_type(annotation, converter=self) - self._logical_type_registry.register_logical_type(lt) - finally: - self._in_progress.discard(annotation) - - return lt.get_arrow_extension_type() - - raise ValueError(f"Unsupported annotation: {annotation!r}") - -def _find_factory_for_class( - self, - python_type: type, -) -> "LogicalTypeFactoryProtocol | None": - """Find the most-specific registered factory for ``python_type``. - - Walks ``python_type.__mro__`` and returns the first factory in - ``_python_class_factories`` whose ``supports_class(python_type)`` returns True. - Falls back to an ``issubclass`` scan for ABC-registered factories. - - Args: - python_type: Concrete Python class to find a factory for. - - Returns: - The matching ``LogicalTypeFactoryProtocol``, or ``None`` if none found. - """ - factories = self._logical_type_registry._python_class_factories - - # MRO walk — most-specific base first - for base in python_type.__mro__: - factory = factories.get(base) - if factory is not None: - if hasattr(factory, "supports_class") and factory.supports_class(python_type): - return factory - elif not hasattr(factory, "supports_class"): - # Factories without supports_class are treated as unconditional matches - return factory - - # issubclass fallback for ABC-registered factories - for base, factory in factories.items(): - try: - if issubclass(python_type, base): - if hasattr(factory, "supports_class"): - if factory.supports_class(python_type): - return factory - else: - return factory - except TypeError: - continue - - return None -``` - -Also add `_in_progress: set[type] = set()` to `__init__`: - -```python -# In __init__, after the existing cache initializations: -self._in_progress: set[type] = set() -``` - -And add `TYPE_CHECKING` import for `LogicalTypeFactoryProtocol`: -```python -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.extension_types.registry import LogicalTypeRegistry - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -k "register_python_class" 2>&1 | tail -30 -``` -Expected: All `register_python_class` tests pass. - -- [ ] **Step 5: Run full test suite for this module** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v 2>&1 | tail -20 -``` -Expected: Existing tests still pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "feat(universal_converter): add register_python_class with recursive traversal, factory dispatch, and cycle detection" -``` - ---- - -## Task 4: Add `register_storage_type` to `UniversalTypeConverter` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Modify: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Write failing tests** - -Add to `tests/test_semantic_types/test_universal_converter.py`: - -```python -# ── register_storage_type tests ────────────────────────────────────────────── - -def test_register_storage_type_primitive_int(): - converter = _make_converter() - assert converter.register_storage_type(pa.int64()) == pa.int64() - - -def test_register_storage_type_primitive_large_string(): - converter = _make_converter() - assert converter.register_storage_type(pa.large_string()) == pa.large_string() - - -def test_register_storage_type_extension_type_registry_hit(): - """An already-registered extension type is returned unchanged (no-op).""" - converter = _make_converter() - # orcapod.uuid is pre-registered in the builtin registry - from orcapod.extension_types.builtin_logical_types import LogicalUUID - uuid_ext = LogicalUUID().get_arrow_extension_type() - result = converter.register_storage_type(uuid_ext) - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "orcapod.uuid" - - -def test_register_storage_type_struct_recurses(): - """Structs are traversed field by field; resolved field types are returned.""" - converter = _make_converter() - struct_type = pa.struct([pa.field("name", pa.large_string()), pa.field("count", pa.int64())]) - result = converter.register_storage_type(struct_type) - assert pa.types.is_struct(result) - assert result.field("name").type == pa.large_string() - assert result.field("count").type == pa.int64() - - -def test_register_storage_type_large_list_recurses(): - converter = _make_converter() - list_type = pa.large_list(pa.int32()) - result = converter.register_storage_type(list_type) - assert pa.types.is_large_list(result) - assert result.value_type == pa.int32() - - -def test_register_storage_type_extension_miss_dispatches_to_factory(): - """An unregistered extension type triggers factory.reconstruct_from_arrow.""" - import json - import uuid as _u - import polars as pl - - ext_name = f"test.reconstruct.{_u.uuid4().hex[:8]}" - category = "test.reconstruct" - metadata = json.dumps({"category": category}).encode() - ArrowExt = make_arrow_extension_type(ext_name, pa.large_string(), metadata=metadata) - PolarsExt = make_polars_extension_type(ext_name, pa.large_string()) - - class _LT: - logical_type_name = ext_name - python_type = str - def get_arrow_extension_type(self): return ArrowExt() - def get_polars_extension_type(self): return PolarsExt() - def python_to_storage(self, v, c=None): return str(v) - def storage_to_python(self, v, c=None): return v - - class _Factory: - def supports_class(self, t): return False - def create_for_python_type(self, t, converter): pass - def reconstruct_from_arrow(self, name, storage_type, meta, converter): - return _LT() - - registry = _make_registry_with_builtins() - registry.register_logical_type_factory(_Factory(), category=category) - converter = _make_converter(registry) - - ext_instance = ArrowExt() - result = converter.register_storage_type(ext_instance) - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == ext_name - - # Second call: registry hit → same result, factory NOT called again - result2 = converter.register_storage_type(ext_instance) - assert result2.extension_name == ext_name - - -def test_register_storage_type_nested_struct_with_extension(): - """Extension type nested inside a struct field is resolved bottom-up.""" - import json - import uuid as _u - import polars as pl - - ext_name = f"test.nested.{_u.uuid4().hex[:8]}" - category = "test.nested" - metadata = json.dumps({"category": category}).encode() - ArrowExt = make_arrow_extension_type(ext_name, pa.large_string(), metadata=metadata) - PolarsExt = make_polars_extension_type(ext_name, pa.large_string()) - - class _LT: - logical_type_name = ext_name - python_type = str - def get_arrow_extension_type(self): return ArrowExt() - def get_polars_extension_type(self): return PolarsExt() - def python_to_storage(self, v, c=None): return str(v) - def storage_to_python(self, v, c=None): return v - - class _Factory: - def supports_class(self, t): return False - def create_for_python_type(self, t, converter): pass - def reconstruct_from_arrow(self, name, storage_type, meta, converter): - return _LT() - - registry = _make_registry_with_builtins() - registry.register_logical_type_factory(_Factory(), category=category) - converter = _make_converter(registry) - - ext_instance = ArrowExt() - struct_with_ext = pa.struct([pa.field("id", pa.int64()), pa.field("tag", ext_instance)]) - result = converter.register_storage_type(struct_with_ext) - - assert pa.types.is_struct(result) - assert result.field("id").type == pa.int64() - assert isinstance(result.field("tag").type, pa.ExtensionType) - assert result.field("tag").type.extension_name == ext_name -``` - -- [ ] **Step 2: Run to confirm failures** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -k "register_storage_type" 2>&1 | tail -30 -``` -Expected: AttributeError — `register_storage_type` not defined. - -- [ ] **Step 3: Implement `register_storage_type` and `_ensure_extension_type_info` in `UniversalTypeConverter`** - -```python -def register_storage_type(self, arrow_type: "pa.DataType") -> "pa.DataType": - """Register extension types found in ``arrow_type`` and return the resolved type. - - Traverses Arrow types recursively in a bottom-up manner: - - Primitives are returned unchanged. - - ``pa.ExtensionType`` instances that are already registered are returned as-is. - - Unregistered extension types: the storage type is resolved first (bottom-up), - then the factory dispatches on the ``"category"`` metadata key. - - Structs: each field's type is resolved; a new struct with resolved fields is returned. - - Lists: the value type is resolved; a new list type with the resolved value is returned. - - Args: - arrow_type: An Arrow type to traverse and register. - - Returns: - The resolved Arrow type with extension types embedded. - """ - # Extension type - if isinstance(arrow_type, pa.ExtensionType): - ext_name = arrow_type.extension_name - if self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_arrow_extension_name(ext_name) - if lt is not None: - return lt.get_arrow_extension_type() - # Registry miss — extract info and register - raw_meta = arrow_type.__arrow_ext_serialize__() - ext_meta = raw_meta if raw_meta else None - resolved_storage = self.register_storage_type(arrow_type.storage_type) - return self._ensure_extension_type_info(ext_name, ext_meta, resolved_storage) - - # Struct type — recurse into each field - if pa.types.is_struct(arrow_type): - resolved_fields = [] - for i in range(arrow_type.num_fields): - field = arrow_type.field(i) - resolved_type = self.register_storage_type(field.type) - resolved_fields.append(pa.field(field.name, resolved_type, nullable=field.nullable)) - return pa.struct(resolved_fields) - - # Large list type - if pa.types.is_large_list(arrow_type): - resolved_value = self.register_storage_type(arrow_type.value_type) - return pa.large_list(resolved_value) - - # List type - if pa.types.is_list(arrow_type): - resolved_value = self.register_storage_type(arrow_type.value_type) - return pa.list_(resolved_value) - - # All other types (primitives, timestamps, binary, etc.) — return as-is - return arrow_type - -def _ensure_extension_type_info( - self, - arrow_extension_name: str, - extension_metadata: bytes | None, - storage_type: "pa.DataType", -) -> "pa.DataType": - """Register an extension type from (name, metadata, storage_type) info. - - Called by ``register_storage_type`` for in-memory ``pa.ExtensionType`` objects, - and by ``register_discovered_extensions`` for the field-metadata (Parquet) channel. - The ``storage_type`` must already be resolved (nested extension types registered). - - Args: - arrow_extension_name: Arrow extension name (``ARROW:extension:name``). - extension_metadata: Raw metadata bytes, expected to be UTF-8 JSON with - at least a ``"category"`` key. ``None`` or empty bytes if absent. - storage_type: Underlying Arrow storage type (already bottom-up resolved). - - Returns: - The Arrow extension type after registration. - - Raises: - ValueError: If metadata is missing, malformed, lacks ``"category"``, or - no factory is registered for the category. - """ - import json as _json - - if self._logical_type_registry is None: - raise ValueError( - f"No LogicalTypeRegistry configured — cannot register extension type " - f"{arrow_extension_name!r}." - ) - - # Registry hit — already registered - lt = self._logical_type_registry.get_by_arrow_extension_name(arrow_extension_name) - if lt is not None: - return lt.get_arrow_extension_type() - - # Missing metadata — cannot auto-register - if not extension_metadata: - raise ValueError( - f"Extension type {arrow_extension_name!r} has no extension metadata. " - f"Types without a metadata category tag cannot be auto-registered via a factory. " - f"Pre-register them explicitly via converter.register_logical_type(lt)." - ) - - # Parse JSON metadata - try: - metadata_dict = _json.loads(extension_metadata.decode("utf-8")) - except (UnicodeDecodeError, _json.JSONDecodeError) as exc: - raise ValueError( - f"Extension type {arrow_extension_name!r} has metadata that is not valid " - f"UTF-8 JSON: {extension_metadata!r}. Parse error: {exc}." - ) from exc - - if not isinstance(metadata_dict, dict): - raise ValueError( - f"Extension type {arrow_extension_name!r} metadata decoded to a non-object " - f"JSON value: {metadata_dict!r}." - ) - - if "category" not in metadata_dict: - raise ValueError( - f"Extension type {arrow_extension_name!r} metadata has no \"category\" key: " - f"{metadata_dict}." - ) - - category = metadata_dict["category"] - if not isinstance(category, str): - raise ValueError( - f"Extension type {arrow_extension_name!r} metadata \"category\" is not a " - f"string: {category!r}." - ) - - # Look up factory by category - factory = self._logical_type_registry._category_factories.get(category) - if factory is None: - raise ValueError( - f"No LogicalTypeFactory registered for category {category!r}. " - f"Cannot register extension type {arrow_extension_name!r}." - ) - - # Reconstruct and register - logical_type = factory.reconstruct_from_arrow( - arrow_extension_name, storage_type, metadata_dict, converter=self - ) - self._logical_type_registry.register_logical_type(logical_type) - return logical_type.get_arrow_extension_type() -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -k "register_storage_type" 2>&1 | tail -30 -``` -Expected: All pass. - -- [ ] **Step 5: Run full converter test suite** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v 2>&1 | tail -10 -``` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "feat(universal_converter): add register_storage_type with bottom-up recursive traversal" -``` - ---- - -## Task 5: Add `python_to_storage`, `storage_to_python`, and registration pass-throughs; update converter dispatch - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Modify: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Write failing tests** - -```python -# ── python_to_storage / storage_to_python / pass-through tests ────────────── - -def test_python_to_storage_for_registered_type(): - """python_to_storage uses the logical type's converter for registered types.""" - converter = _make_converter() - import pathlib - result = converter.python_to_storage(pathlib.Path("/tmp/bar"), pathlib.Path) - assert result == "/tmp/bar" - - -def test_storage_to_python_for_registered_type(): - converter = _make_converter() - import pathlib - result = converter.storage_to_python("/tmp/bar", pathlib.Path) - assert isinstance(result, pathlib.Path) - assert result == pathlib.Path("/tmp/bar") - - -def test_python_to_storage_for_int(): - converter = _make_converter() - assert converter.python_to_storage(42, int) == 42 - - -def test_register_logical_type_passthrough(): - from orcapod.extension_types.builtin_logical_types import LogicalPath - registry = LogicalTypeRegistry() - converter = UniversalTypeConverter(logical_type_registry=registry) - lt = LogicalPath() - converter.register_logical_type(lt) - assert registry.get_by_python_type(import_pathlib_path()) is lt - - -def import_pathlib_path(): - import pathlib; return pathlib.Path - - -def test_register_logical_type_factory_passthrough(): - import uuid as _u - import polars as pl - - class _Factory: - def supports_class(self, t): return False - def create_for_python_type(self, t, converter): pass - def reconstruct_from_arrow(self, name, storage, meta, converter): pass - - registry = LogicalTypeRegistry() - converter = UniversalTypeConverter(logical_type_registry=registry) - factory = _Factory() - converter.register_logical_type_factory(factory, category="test.cat") - assert registry._category_factories.get("test.cat") is factory -``` - -- [ ] **Step 2: Run to confirm failures** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -v -k "python_to_storage or storage_to_python or passthrough" 2>&1 | tail -20 -``` - -- [ ] **Step 3: Add methods to `UniversalTypeConverter`** - -```python -def python_to_storage(self, value: Any, annotation: Any) -> Any: - """Convert a Python value to its Arrow storage representation. - - Thin wrapper over ``get_python_to_arrow_converter`` for use by - ``DataclassLogicalType`` and other logical types that delegate per-field - conversion back to the converter. - - Args: - value: A Python object. - annotation: The Python type annotation for ``value``. - - Returns: - A value in Arrow storage format. - """ - converter_fn = self.get_python_to_arrow_converter(annotation) - return converter_fn(value) - -def storage_to_python(self, storage_value: Any, annotation: Any) -> Any: - """Convert an Arrow storage value back to a Python object. - - Args: - storage_value: A scalar or element from an Arrow storage array. - annotation: The Python type annotation to convert back to. - - Returns: - A Python object of the type described by ``annotation``. - """ - arrow_type = self.python_type_to_arrow_type(annotation) - converter_fn = self.get_arrow_to_python_converter(arrow_type) - return converter_fn(storage_value) - -def register_logical_type(self, lt: "LogicalTypeProtocol") -> None: - """Register a ``LogicalTypeProtocol`` instance. - - Pass-through to the internal ``LogicalTypeRegistry``. - - Args: - lt: The logical type to register. - """ - if self._logical_type_registry is None: - raise ValueError("No LogicalTypeRegistry configured on this converter.") - self._logical_type_registry.register_logical_type(lt) - -def register_logical_type_factory( - self, - factory: "LogicalTypeFactoryProtocol", - *, - category: "str | None" = None, - python_bases: "Iterable[type]" = (), -) -> None: - """Register a ``LogicalTypeFactoryProtocol`` instance. - - Pass-through to the internal ``LogicalTypeRegistry``. - - Args: - factory: The factory to register. - category: If given, registers factory as the read-side handler for - Arrow extension types with this ``"category"`` metadata value. - python_bases: Zero or more Python base classes to register as write-side - dispatch keys for this factory. - """ - if self._logical_type_registry is None: - raise ValueError("No LogicalTypeRegistry configured on this converter.") - self._logical_type_registry.register_logical_type_factory( - factory, category=category, python_bases=python_bases - ) -``` - -Also add `Iterable` to the imports in `universal_converter.py`: -```python -from collections.abc import Callable, Iterable, Mapping -``` - -And add TYPE_CHECKING imports: -```python -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.extension_types.registry import LogicalTypeRegistry - from orcapod.extension_types.protocols import LogicalTypeFactoryProtocol, LogicalTypeProtocol -``` - -- [ ] **Step 4: Update `_create_python_to_arrow_converter` to pass `converter=self`** - -In `_create_python_to_arrow_converter`, find this block: -```python -if self._logical_type_registry is not None and isinstance(python_type, type): - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - return lt.python_to_storage -``` - -Replace with: -```python -if self._logical_type_registry is not None and isinstance(python_type, type): - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - _lt = lt - _self = self - return lambda value: _lt.python_to_storage(value, _self) -``` - -- [ ] **Step 5: Update `_create_arrow_to_python_converter` to pass `converter=self`** - -In `_create_arrow_to_python_converter`, find: -```python -if isinstance(arrow_type, pa.ExtensionType) and self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_arrow_extension_name( - arrow_type.extension_name - ) - if lt is not None: - return lt.storage_to_python -``` - -Replace with: -```python -if isinstance(arrow_type, pa.ExtensionType) and self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_arrow_extension_name( - arrow_type.extension_name - ) - if lt is not None: - _lt = lt - _self = self - return lambda storage_value: _lt.storage_to_python(storage_value, _self) -``` - -- [ ] **Step 6: Run tests** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py tests/test_extension_types/test_builtin_logical_types.py -v 2>&1 | tail -20 -``` -Expected: All pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "feat(universal_converter): add python_to_storage, storage_to_python, and registration pass-throughs; wire converter=self into logical type dispatch" -``` - ---- - -## Task 6: Simplify `ensure_types_registered_for_schemas` + remove `ensure_*` from registry - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Modify: `src/orcapod/extension_types/registry.py` -- Modify: `tests/test_extension_types/test_registry.py` - -- [ ] **Step 1: Update `ensure_types_registered_for_schemas` in `UniversalTypeConverter`** - -Replace the existing method: - -```python -def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: - """Ensure a LogicalType is registered for every annotation in schemas. - - Calls ``register_python_class`` for each annotation, which recursively - resolves nested types and synthesises via factory if needed. - When no ``LogicalTypeRegistry`` is configured, this is a no-op. - - Args: - *schemas: One or more ``Schema`` mappings (column name → Python type). - - Raises: - TypeError: If a leaf class has no registered ``LogicalType`` and - no registered factory covers it. - """ - if self._logical_type_registry is None: - return - for schema in schemas: - for annotation in schema.values(): - self.register_python_class(annotation) -``` - -- [ ] **Step 2: Run existing ensure_types tests to verify nothing breaks** - -```bash -uv run pytest tests/ -v -k "ensure_types" 2>&1 | tail -20 -``` -Expected: Pass. - -- [ ] **Step 3: Find and update registry tests that test `ensure_*` methods** - -Check which tests in `test_registry.py` test `ensure_logical_type_for_python_class` and `ensure_extension_type`: - -```bash -grep -n "ensure_logical_type\|ensure_extension_type" tests/test_extension_types/test_registry.py -``` - -- [ ] **Step 4: Remove `ensure_*` tests from `test_registry.py` and add converter pass-through tests** - -Remove any test functions that directly test `ensure_logical_type_for_python_class` or `ensure_extension_type` on the registry (they are removed from the public API). - -Add this test to `test_registry.py`: - -```python -def test_registry_does_not_expose_ensure_methods(): - """ensure_logical_type_for_python_class and ensure_extension_type are removed.""" - registry = LogicalTypeRegistry() - assert not hasattr(registry, "ensure_logical_type_for_python_class") - assert not hasattr(registry, "ensure_extension_type") -``` - -- [ ] **Step 5: Remove `ensure_logical_type_for_python_class` and `ensure_extension_type` from `registry.py`** - -In `src/orcapod/extension_types/registry.py`, delete the `ensure_extension_type` method (lines ~355-467) and the `ensure_logical_type_for_python_class` method (lines ~469-577). - -The public surface retained: `register_logical_type`, `register_logical_type_factory`, `get_by_python_type`, `get_by_arrow_extension_name`, `get_by_logical_name`. - -- [ ] **Step 6: Run tests** - -```bash -uv run pytest tests/test_extension_types/test_registry.py tests/test_semantic_types/ -v 2>&1 | tail -20 -``` -Expected: All pass (ensure_* tests replaced). - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py src/orcapod/extension_types/registry.py tests/test_extension_types/test_registry.py -git commit -m "refactor(registry): remove ensure_* methods; simplify ensure_types_registered_for_schemas to use register_python_class" -``` - ---- - -## Task 7: Create `DataclassLogicalType` in `extension_types/dataclass_handler.py` - -**Files:** -- Create: `src/orcapod/extension_types/dataclass_handler.py` -- Create: `tests/test_extension_types/test_dataclass_handler.py` - -- [ ] **Step 1: Write failing tests for `DataclassLogicalType`** - -Create `tests/test_extension_types/test_dataclass_handler.py`: - -```python -"""Tests for DataclassLogicalType and DataclassHandlerFactory.""" - -from __future__ import annotations - -import dataclasses -import uuid as _uuid_module -from typing import Any - -import pyarrow as pa -import pytest - - -# ── Helpers ───────────────────────────────────────────────────────────────── - -class _StubConverter: - """Minimal converter stub for DataclassLogicalType tests.""" - - def python_to_storage(self, value, annotation): - if annotation is str: - return str(value) - if annotation is int: - return int(value) - return value - - def storage_to_python(self, storage_value, annotation): - if annotation is str: - return str(storage_value) - if annotation is int: - return int(storage_value) - return storage_value - - def register_python_class(self, annotation): - if annotation is str: - return pa.large_string() - if annotation is int: - return pa.int64() - raise ValueError(f"No mapping for {annotation}") - - -# ── DataclassLogicalType tests ─────────────────────────────────────────────── - -def test_dataclass_logical_type_is_importable(): - from orcapod.extension_types.dataclass_handler import DataclassLogicalType - assert DataclassLogicalType is not None - - -def test_dataclass_logical_type_protocol_conformance(): - from orcapod.extension_types.dataclass_handler import DataclassLogicalType - from orcapod.extension_types.protocols import LogicalTypeProtocol - - @dataclasses.dataclass - class _MyDC: - name: str - count: int - - storage = pa.struct([pa.field("name", pa.large_string()), pa.field("count", pa.int64())]) - field_annotations = [("name", str), ("count", int)] - lt = DataclassLogicalType( - logical_name="tests.MyDC", - python_type=_MyDC, - storage_type=storage, - field_annotations=field_annotations, - ) - assert isinstance(lt, LogicalTypeProtocol) - - -def test_dataclass_logical_type_python_to_storage(): - from orcapod.extension_types.dataclass_handler import DataclassLogicalType - - @dataclasses.dataclass - class _Point: - x: int - y: int - - storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - lt = DataclassLogicalType("tests.Point", _Point, storage, [("x", int), ("y", int)]) - converter = _StubConverter() - - result = lt.python_to_storage(_Point(x=3, y=7), converter) - assert result == {"x": 3, "y": 7} - - -def test_dataclass_logical_type_storage_to_python(): - from orcapod.extension_types.dataclass_handler import DataclassLogicalType - - @dataclasses.dataclass - class _Point: - x: int - y: int - - storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - lt = DataclassLogicalType("tests.Point", _Point, storage, [("x", int), ("y", int)]) - converter = _StubConverter() - - result = lt.storage_to_python({"x": 3, "y": 7}, converter) - assert isinstance(result, _Point) - assert result.x == 3 - assert result.y == 7 - - -def test_dataclass_logical_type_logical_type_name(): - from orcapod.extension_types.dataclass_handler import DataclassLogicalType - - @dataclasses.dataclass - class _Foo: - val: str - - storage = pa.struct([pa.field("val", pa.large_string())]) - lt = DataclassLogicalType("mymod.Foo", _Foo, storage, [("val", str)]) - assert lt.logical_type_name == "mymod.Foo" - - -def test_dataclass_logical_type_python_type(): - from orcapod.extension_types.dataclass_handler import DataclassLogicalType - - @dataclasses.dataclass - class _Bar: - val: str - - storage = pa.struct([pa.field("val", pa.large_string())]) - lt = DataclassLogicalType("mymod.Bar", _Bar, storage, [("val", str)]) - assert lt.python_type is _Bar -``` - -- [ ] **Step 2: Run to confirm failures** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_handler.py -v 2>&1 | tail -20 -``` -Expected: ImportError — `dataclass_handler` does not exist. - -- [ ] **Step 3: Create `src/orcapod/extension_types/dataclass_handler.py`** - -```python -"""DataclassLogicalType and DataclassHandlerFactory. - -Provides the ``DataclassLogicalType`` logical type implementation and the -``DataclassHandlerFactory`` that synthesises and reconstructs ``DataclassLogicalType`` -instances for Python dataclasses. - -Write path (``create_for_python_type``): - Iterates dataclass fields, delegates field Arrow-type resolution to the converter - via ``register_python_class``, and returns a ``DataclassLogicalType`` backed by - a ``pa.struct`` extension type. - -Read path (``reconstruct_from_arrow``): - Imports the dataclass by fully-qualified class name, resolves field annotations - against the (already bottom-up resolved) storage type, and returns a - ``DataclassLogicalType``. - -Category tag: ``"orcapod.dataclass"`` -""" - -from __future__ import annotations - -import dataclasses -import importlib -import json -import logging -from typing import TYPE_CHECKING, Any - -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from orcapod.extension_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) - -#: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. -DATACLASS_CATEGORY = "orcapod.dataclass" - - -class DataclassLogicalType: - """Logical type binding a Python dataclass to its Arrow extension type representation. - - Stores the dataclass's fully-qualified class name as the Arrow extension name - and a ``pa.struct`` of the dataclass fields as the storage type. - - No Arrow-type reasoning lives here — all field-type resolution is owned by the - converter and completed before this object is constructed. - - Args: - logical_name: Fully-qualified class name (e.g. ``"mymodule.sub.MyData"``). - Used as both the logical type name and the Arrow extension name. - python_type: The Python dataclass ``type`` object. - storage_type: The Arrow ``pa.StructType`` for the dataclass fields. - field_annotations: Ordered list of ``(field_name, python_annotation)`` pairs - matching the fields in ``storage_type``. - - Example: - >>> lt = DataclassLogicalType( - ... "mymod.Point", Point, - ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), - ... [("x", int), ("y", int)], - ... ) - >>> lt.python_to_storage(Point(1, 2), converter) - {"x": 1, "y": 2} - """ - - def __init__( - self, - logical_name: str, - python_type: type, - storage_type: "pa.StructType", - field_annotations: list[tuple[str, Any]], - ) -> None: - self._logical_name = logical_name - self._python_type = python_type - self._storage_type = storage_type - self._field_annotations = field_annotations - - _metadata = json.dumps({"category": DATACLASS_CATEGORY}).encode("utf-8") - self._arrow_ext_class = make_arrow_extension_type( - logical_name, storage_type, metadata=_metadata - ) - self._arrow_ext: "pa.ExtensionType | None" = None - self._polars_ext_class = make_polars_extension_type(logical_name, storage_type) - self._polars_ext: "pl.BaseExtension | None" = None - - @property - def logical_type_name(self) -> str: - """Fully-qualified class name used as the logical type identifier.""" - return self._logical_name - - @property - def python_type(self) -> type: - """The Python dataclass type this logical type represents.""" - return self._python_type - - def get_arrow_extension_type(self) -> "pa.ExtensionType": - """Return the Arrow extension type for this dataclass. - - Returns: - A cached ``pa.ExtensionType`` instance with ``extension_name`` equal to - the fully-qualified class name and ``storage_type`` equal to the struct - of the dataclass fields. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> "pl.BaseExtension": - """Return the Polars extension type for this dataclass. - - Returns: - A cached ``pl.BaseExtension`` instance. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext - - def python_to_storage(self, value: Any, converter: "TypeConverterProtocol") -> dict[str, Any]: - """Convert a dataclass instance to an Arrow-compatible struct dict. - - Iterates ``_field_annotations`` and delegates each field's conversion to - ``converter.python_to_storage``. - - Args: - value: A dataclass instance of type ``python_type``. - converter: The active converter for per-field delegation. - - Returns: - A dict mapping field names to their Arrow storage values. - """ - return { - name: converter.python_to_storage(getattr(value, name), annotation) - for name, annotation in self._field_annotations - } - - def storage_to_python(self, storage_value: Any, converter: "TypeConverterProtocol") -> Any: - """Reconstruct a dataclass instance from an Arrow struct dict. - - Args: - storage_value: A dict mapping field names to Arrow storage values. - converter: The active converter for per-field delegation. - - Returns: - A dataclass instance of type ``python_type``. - """ - kwargs = { - name: converter.storage_to_python(storage_value[name], annotation) - for name, annotation in self._field_annotations - } - return self._python_type(**kwargs) - - -class DataclassHandlerFactory: - """Stateless factory that synthesises and reconstructs ``DataclassLogicalType`` instances. - - **Write path** (``create_for_python_type``): derives Arrow struct type from the - dataclass fields by delegating to ``converter.register_python_class`` per field. - - **Read path** (``reconstruct_from_arrow``): imports the dataclass by FQCN, matches - fields against the already-resolved ``storage_type``, and returns a - ``DataclassLogicalType``. - - Category tag: ``"orcapod.dataclass"`` - Register with:: - - converter.register_logical_type_factory( - DataclassHandlerFactory(), - category="orcapod.dataclass", - python_bases=[object], - ) - - Example: - >>> factory = DataclassHandlerFactory() - >>> factory.supports_class(MyDataclass) - True - >>> factory.supports_class(str) - False - """ - - def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a dataclass. - - Args: - python_type: Any Python type. - - Returns: - True if ``dataclasses.is_dataclass(python_type)`` is True. - """ - return dataclasses.is_dataclass(python_type) and isinstance(python_type, type) - - def create_for_python_type( - self, - python_type: type, - converter: "TypeConverterProtocol", - ) -> DataclassLogicalType: - """Synthesise a ``DataclassLogicalType`` for a Python dataclass (write path). - - Derives the FQCN, obtains type hints, and resolves each field's Arrow type - via ``converter.register_python_class``. Rejects local / unnamed classes. - - Args: - python_type: A Python dataclass type. - converter: The active converter for field-type resolution. - - Returns: - A ``DataclassLogicalType`` ready for registration. - - Raises: - ValueError: If ``python_type`` is a local class (no stable FQCN) or - has a ``__qualname__`` that contains ``""``. - """ - import typing - - fqcn = f"{python_type.__module__}.{python_type.__qualname__}" - if "" in fqcn or not python_type.__module__ or python_type.__module__ == "__main__": - pass # allow __main__ classes but reject proper locals - if "" in fqcn: - raise ValueError( - f"Cannot register local class {python_type!r} as a DataclassLogicalType — " - f"local classes have no stable fully-qualified class name and cannot be " - f"reconstructed on read. Define the dataclass at module level." - ) - - try: - hints = typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {python_type!r}: {exc}" - ) from exc - - arrow_fields = [] - field_annotations = [] - for field in dataclasses.fields(python_type): - if not field.init: - continue - annotation = hints.get(field.name, Any) - arrow_type = converter.register_python_class(annotation) - arrow_fields.append(pa.field(field.name, arrow_type)) - field_annotations.append((field.name, annotation)) - - storage_type = pa.struct(arrow_fields) - logger.debug("DataclassHandlerFactory: synthesised %r for %r", fqcn, python_type) - return DataclassLogicalType(fqcn, python_type, storage_type, field_annotations) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: "pa.DataType", - metadata: dict[str, Any], - converter: "TypeConverterProtocol", - ) -> DataclassLogicalType: - """Reconstruct a ``DataclassLogicalType`` from Arrow schema metadata (read path). - - Imports the dataclass from its FQCN (``arrow_extension_name``), then matches - the dataclass field annotations against the fields in ``storage_type``. - ``storage_type`` is already bottom-up resolved by ``register_storage_type`` - before this method is called. - - Args: - arrow_extension_name: FQCN of the dataclass (Arrow extension name). - storage_type: Already-resolved ``pa.StructType`` for the dataclass fields. - metadata: Full parsed metadata JSON dict (always contains ``"category"``). - converter: The active converter (not needed here but required by protocol). - - Returns: - A ``DataclassLogicalType`` ready for registration. - - Raises: - ImportError: If the class cannot be imported from ``arrow_extension_name``. - ValueError: If ``storage_type`` is not a struct type. - """ - import typing - - if not pa.types.is_struct(storage_type): - raise ValueError( - f"DataclassHandlerFactory.reconstruct_from_arrow: expected a struct " - f"storage type for {arrow_extension_name!r}, got {storage_type!r}." - ) - - # Import class from FQCN using longest-prefix module walk - cls = _import_from_fqcn(arrow_extension_name) - - try: - hints = typing.get_type_hints(cls) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {cls!r}: {exc}" - ) from exc - - field_annotations = [] - for field in dataclasses.fields(cls): - if not field.init: - continue - annotation = hints.get(field.name, Any) - field_annotations.append((field.name, annotation)) - - logger.debug( - "DataclassHandlerFactory: reconstructed %r from Arrow", arrow_extension_name - ) - return DataclassLogicalType( - arrow_extension_name, cls, storage_type, field_annotations - ) - - -def _import_from_fqcn(fqcn: str) -> type: - """Import a class from its fully-qualified class name. - - Tries module prefixes from longest to shortest. For example, for - ``"mypackage.sub.MyClass"``, tries ``importlib.import_module("mypackage.sub")`` - then ``getattr(module, "MyClass")``. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyClass"``. - - Returns: - The imported class. - - Raises: - ImportError: If no valid module+attribute split can be found. - """ - parts = fqcn.rsplit(".", 1) - if len(parts) != 2: - raise ImportError(f"Cannot import from FQCN {fqcn!r}: no module separator found.") - - module_path, class_name = parts - try: - module = importlib.import_module(module_path) - cls = getattr(module, class_name) - if not dataclasses.is_dataclass(cls) or not isinstance(cls, type): - raise ImportError( - f"{class_name!r} in {module_path!r} is not a dataclass type." - ) - return cls - except (ImportError, AttributeError, ModuleNotFoundError) as exc: - raise ImportError( - f"Cannot import dataclass from FQCN {fqcn!r}: {exc}" - ) from exc -``` - -- [ ] **Step 4: Run dataclass logical type tests** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_handler.py -v -k "DataclassLogicalType or logical_type" 2>&1 | tail -30 -``` -Expected: All DataclassLogicalType tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/dataclass_handler.py tests/test_extension_types/test_dataclass_handler.py -git commit -m "feat(dataclass_handler): implement DataclassLogicalType" -``` - ---- - -## Task 8: `DataclassHandlerFactory` write path tests + verification - -**Files:** -- Modify: `tests/test_extension_types/test_dataclass_handler.py` -- Modify: `src/orcapod/extension_types/dataclass_handler.py` (fixes only) - -- [ ] **Step 1: Add factory write-path tests** - -```python -# Add to tests/test_extension_types/test_dataclass_handler.py - -def _make_full_converter(): - """Make a UniversalTypeConverter with builtin types + DataclassHandlerFactory.""" - from orcapod.extension_types.builtin_logical_types import LogicalPath, LogicalUUID, LogicalUPath - from orcapod.extension_types.registry import LogicalTypeRegistry - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory, DATACLASS_CATEGORY - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - registry = LogicalTypeRegistry(logical_types=[LogicalPath(), LogicalUUID(), LogicalUPath()]) - factory = DataclassHandlerFactory() - registry.register_logical_type_factory(factory, category=DATACLASS_CATEGORY, python_bases=[object]) - return UniversalTypeConverter(logical_type_registry=registry) - - -def test_factory_supports_class_dataclass(): - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - @dataclasses.dataclass - class _Dummy: - x: int - - factory = DataclassHandlerFactory() - assert factory.supports_class(_Dummy) is True - - -def test_factory_supports_class_non_dataclass(): - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - factory = DataclassHandlerFactory() - assert factory.supports_class(str) is False - assert factory.supports_class(int) is False - - -def test_factory_create_flat_dataclass(): - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory, DataclassLogicalType - - @dataclasses.dataclass - class _Flat: - name: str - count: int - - factory = DataclassHandlerFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_Flat, converter=converter) - - assert isinstance(lt, DataclassLogicalType) - storage = lt.get_arrow_extension_type().storage_type - assert pa.types.is_struct(storage) - assert storage.field("name").type == pa.large_string() - assert storage.field("count").type == pa.int64() - - -def test_factory_create_dataclass_with_uuid_field(): - """UUID field → orcapod.uuid extension type in storage struct.""" - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - @dataclasses.dataclass - class _WithUUID: - id: _uuid_module.UUID - label: str - - factory = DataclassHandlerFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_WithUUID, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - id_field_type = storage.field("id").type - assert isinstance(id_field_type, pa.ExtensionType) - assert id_field_type.extension_name == "orcapod.uuid" - - -def test_factory_create_dataclass_with_list_field(): - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - @dataclasses.dataclass - class _WithList: - tags: list[str] - count: int - - factory = DataclassHandlerFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_WithList, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - assert pa.types.is_large_list(storage.field("tags").type) - assert storage.field("tags").type.value_type == pa.large_string() - - -def test_factory_create_dataclass_with_dict_field(): - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - @dataclasses.dataclass - class _WithDict: - meta: dict[str, int] - - factory = DataclassHandlerFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_WithDict, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - meta_type = storage.field("meta").type - assert pa.types.is_large_list(meta_type) - assert pa.types.is_struct(meta_type.value_type) - field_names = {meta_type.value_type.field(i).name for i in range(meta_type.value_type.num_fields)} - assert field_names == {"key", "value"} - - -def test_factory_rejects_local_class(): - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - def _make_local(): - @dataclasses.dataclass - class _Local: - x: int - return _Local - - LocalClass = _make_local() - factory = DataclassHandlerFactory() - converter = _make_full_converter() - with pytest.raises(ValueError, match="local"): - factory.create_for_python_type(LocalClass, converter=converter) - - -def test_register_python_class_dispatches_to_dataclass_factory(): - """register_python_class on a dataclass triggers DataclassHandlerFactory.""" - converter = _make_full_converter() - - @dataclasses.dataclass - class _MyPoint: - x: int - y: int - - # This is a local class — use a module-level one via register_python_class - # For this test, simulate by directly pre-importing: - # We can't use a local class here due to the FQCN check. - # So we test with the UUID field only as a proxy. - result = converter.register_python_class(_uuid_module.UUID) - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "orcapod.uuid" -``` - -- [ ] **Step 2: Run factory write-path tests** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_handler.py -v 2>&1 | tail -30 -``` -Expected: All pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_dataclass_handler.py src/orcapod/extension_types/dataclass_handler.py -git commit -m "test(dataclass_handler): add DataclassHandlerFactory write-path tests" -``` - ---- - -## Task 9: `DataclassHandlerFactory` read path + Arrow round-trip - -**Files:** -- Modify: `tests/test_extension_types/test_dataclass_handler.py` - -- [ ] **Step 1: Add read-path and round-trip tests** - -```python -# Add to tests/test_extension_types/test_dataclass_handler.py - -# ── Module-level dataclass for round-trip tests ────────────────────────────── - -@dataclasses.dataclass -class _RoundTripPoint: - """Module-level dataclass for round-trip testing.""" - x: int - y: int - - -@dataclasses.dataclass -class _RoundTripRecord: - """Module-level dataclass with a UUID field.""" - record_id: _uuid_module.UUID - label: str - - -# ── Read-path tests ─────────────────────────────────────────────────────────── - -def test_factory_reconstruct_from_arrow(): - """reconstruct_from_arrow rebuilds the logical type from the Arrow struct.""" - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory, DataclassLogicalType - - storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - metadata = {"category": "orcapod.dataclass"} - fqcn = f"{_RoundTripPoint.__module__}.{_RoundTripPoint.__qualname__}" - - factory = DataclassHandlerFactory() - converter = _make_full_converter() - lt = factory.reconstruct_from_arrow(fqcn, storage, metadata, converter=converter) - - assert isinstance(lt, DataclassLogicalType) - assert lt.python_type is _RoundTripPoint - assert lt.logical_type_name == fqcn - - -def test_factory_reconstruct_from_arrow_invalid_fqcn(): - """ImportError if the FQCN cannot be resolved.""" - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - storage = pa.struct([pa.field("x", pa.int64())]) - factory = DataclassHandlerFactory() - converter = _make_full_converter() - - with pytest.raises(ImportError): - factory.reconstruct_from_arrow( - "nonexistent.module.NoSuchClass", storage, {"category": "orcapod.dataclass"}, converter - ) - - -def test_dataclass_python_to_storage_round_trip(): - """python_to_storage → storage_to_python returns an equivalent dataclass.""" - converter = _make_full_converter() - - # Register _RoundTripPoint via register_python_class - # It's module-level so FQCN is stable - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory, DATACLASS_CATEGORY - factory = DataclassHandlerFactory() - lt = factory.create_for_python_type(_RoundTripPoint, converter=converter) - converter.register_logical_type(lt) - - point = _RoundTripPoint(x=10, y=20) - storage_value = lt.python_to_storage(point, converter) - assert storage_value == {"x": 10, "y": 20} - - reconstructed = lt.storage_to_python(storage_value, converter) - assert isinstance(reconstructed, _RoundTripPoint) - assert reconstructed.x == 10 - assert reconstructed.y == 20 - - -def test_dataclass_with_uuid_round_trip(): - """Round-trip a dataclass with a UUID field through python_to_storage / storage_to_python.""" - from orcapod.extension_types.dataclass_handler import DataclassHandlerFactory - - converter = _make_full_converter() - factory = DataclassHandlerFactory() - lt = factory.create_for_python_type(_RoundTripRecord, converter=converter) - converter.register_logical_type(lt) - - u = _uuid_module.UUID("12345678-1234-5678-1234-567812345678") - record = _RoundTripRecord(record_id=u, label="hello") - - storage_value = lt.python_to_storage(record, converter) - assert storage_value["label"] == "hello" - # UUID stored as bytes - assert storage_value["record_id"] == u.bytes - - reconstructed = lt.storage_to_python(storage_value, converter) - assert isinstance(reconstructed, _RoundTripRecord) - assert reconstructed.record_id == u - assert reconstructed.label == "hello" -``` - -- [ ] **Step 2: Run read-path and round-trip tests** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_handler.py -v 2>&1 | tail -30 -``` -Expected: All pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_dataclass_handler.py -git commit -m "test(dataclass_handler): add DataclassHandlerFactory read-path and Arrow round-trip tests" -``` - ---- - -## Task 10: DataContext cleanup + context wiring - -**Files:** -- Modify: `src/orcapod/contexts/core.py` -- Modify: `src/orcapod/contexts/__init__.py` -- Modify: `src/orcapod/contexts/registry.py` -- Modify: `src/orcapod/contexts/data/v0.1.json` -- Modify: `src/orcapod/contexts/data/schemas/context_schema.json` -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `tests/test_core/function_pod/test_write_side_registration.py` - -- [ ] **Step 1: Remove `logical_type_registry` from `DataContext`** - -In `src/orcapod/contexts/core.py`, remove the `logical_type_registry` field: - -```python -"""Core data structures and exceptions for the OrcaPod context system.""" - -from dataclasses import dataclass - -from orcapod.hashing.semantic_hashing.type_handler_registry import TypeHandlerRegistry -from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - SemanticHasherProtocol, -) -from orcapod.protocols.semantic_types_protocols import TypeConverterProtocol - - -@dataclass -class DataContext: - """Data context containing all versioned components needed for data interpretation. - - Attributes: - context_key: Unique identifier (e.g., "std:v0.1:default") - version: Version string (e.g., "v0.1") - description: Human-readable description - type_converter: Type converter for Python ↔ Arrow conversion and - registration. This is the single public API for all type operations. - arrow_hasher: Arrow table hasher for this context - semantic_hasher: General semantic hasher for this context - type_handler_registry: Registry of TypeHandlerProtocol instances - """ - - context_key: str - version: str - description: str - type_converter: TypeConverterProtocol - arrow_hasher: ArrowHasherProtocol - semantic_hasher: SemanticHasherProtocol - type_handler_registry: TypeHandlerRegistry - - -class ContextValidationError(Exception): - """Raised when context validation fails.""" - pass - - -class ContextResolutionError(Exception): - """Raised when context cannot be resolved.""" - pass -``` - -- [ ] **Step 2: Remove `get_default_logical_type_registry` from `contexts/__init__.py`** - -In `src/orcapod/contexts/__init__.py`: -1. Remove the `from orcapod.extension_types.registry import LogicalTypeRegistry` import -2. Delete the `get_default_logical_type_registry` function -3. Remove `get_default_logical_type_registry` from `__all__` - -- [ ] **Step 3: Update `contexts/registry.py`** - -In `_create_context_from_spec`, remove `logical_type_registry=ref_lut["logical_type_registry"]` from `DataContext(...)` constructor call. Also remove `"logical_type_registry"` from the `required_fields` list: - -```python -required_fields = [ - "context_key", - "version", - "semantic_registry", - "type_converter", - "arrow_hasher", - "semantic_hasher", - "type_handler_registry", - # "logical_type_registry" — removed; registry is internal to type_converter -] -``` - -And update `DataContext(...)` construction: -```python -return DataContext( - context_key=context_key, - version=version, - description=description, - type_converter=ref_lut["type_converter"], - arrow_hasher=ref_lut["arrow_hasher"], - semantic_hasher=ref_lut["semantic_hasher"], - type_handler_registry=ref_lut["type_handler_registry"], - # logical_type_registry removed -) -``` - -- [ ] **Step 4: Update `contexts/data/v0.1.json`** - -Move `logical_type_registry` construction inside `type_converter._config`. Remove `semantic_registry` ref from `type_converter._config`: - -```json -"type_converter": { - "_class": "orcapod.semantic_types.universal_converter.UniversalTypeConverter", - "_config": { - "logical_type_registry": { - "_class": "orcapod.extension_types.registry.LogicalTypeRegistry", - "_config": { - "logical_types": [ - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUUID", - "_config": {} - } - ] - } - } - } -}, -``` - -Also remove the top-level `"logical_type_registry"` key from the JSON file entirely. - -Keep `semantic_registry` at the top level (used by `arrow_hasher`). It's no longer passed to `type_converter`. - -- [ ] **Step 5: Update `contexts/data/schemas/context_schema.json`** - -Remove `"logical_type_registry"` from `"required"` array and from `"properties"`. - -- [ ] **Step 6: Update `extension_types/__init__.py`** docstring to remove the `DataContext.logical_type_registry` access path reference. - -- [ ] **Step 7: Update `test_write_side_registration.py`** - -Update `_make_test_context` to not pass `logical_type_registry`: - -```python -def _make_test_context(registry: LogicalTypeRegistry) -> DataContext: - """Create a DataContext with a fresh converter bound to the given registry.""" - base_ctx = get_default_context() - fresh_converter = UniversalTypeConverter( - logical_type_registry=registry, - ) - return DataContext( - context_key="test", - version="test", - description="test", - type_converter=fresh_converter, - arrow_hasher=base_ctx.arrow_hasher, - semantic_hasher=base_ctx.semantic_hasher, - type_handler_registry=base_ctx.type_handler_registry, - # logical_type_registry removed from DataContext - ) -``` - -Also update the factory stub to use new protocol signatures: -```python -class _Factory: - def supports_class(self, python_type): # new method - return True - def reconstruct_from_arrow(self, name, storage, meta, converter): - return _make_logical_type(object) - def create_for_python_type(self, python_type, converter): # converter param - call_log.append(python_type) - return _make_logical_type(python_type) -``` - -And update `_make_logical_type` builtin logical type stubs to accept converter param: -```python -class _LT: - ... - def python_to_storage(self, v, converter=None): return str(v) - def storage_to_python(self, v, converter=None): return v -``` - -- [ ] **Step 8: Run tests related to contexts and write-side registration** - -```bash -uv run pytest tests/test_core/function_pod/test_write_side_registration.py -v 2>&1 | tail -30 -``` - -```bash -uv run pytest -v -k "context" 2>&1 | tail -20 -``` - -- [ ] **Step 9: Commit** - -```bash -git add src/orcapod/contexts/ tests/test_core/function_pod/test_write_side_registration.py -git commit -m "refactor(contexts): remove logical_type_registry from DataContext; move registry construction inside type_converter config" -``` - ---- - -## Task 11: Update `database_hooks.py` and `ExtensionAwareDatabase` - -**Files:** -- Modify: `src/orcapod/extension_types/database_hooks.py` -- Modify: `src/orcapod/databases/extension_aware_database.py` -- Modify: `tests/test_extension_types/test_database_hooks.py` - -- [ ] **Step 1: Update `register_discovered_extensions` in `database_hooks.py`** - -```python -"""Schema-walking utilities for extension type auto-registration and post-load casting.""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -from orcapod.extension_types.schema_walker import walk_schema - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - -logger = logging.getLogger(__name__) - - -def register_discovered_extensions( - converter: "UniversalTypeConverter | None", - schema: "pa.Schema", -) -> None: - """Register any extension types found in ``schema`` that are not yet known. - - Walks ``schema`` recursively via ``walk_schema`` to discover all Arrow extension - types at any nesting depth (both in-memory and field-metadata channels). - For each discovered type, delegates to ``converter._ensure_extension_type_info``. - - Args: - converter: The ``UniversalTypeConverter`` to use for registration. - If ``None``, this call is a no-op. - schema: The Arrow schema to inspect. - - Raises: - ValueError: Propagated from the converter if an extension type's metadata - has no registered factory or is malformed. - """ - if converter is None: - logger.debug("register_discovered_extensions: no converter provided, skipping") - return - - found = walk_schema(schema) - if not found: - logger.debug("register_discovered_extensions: no extension types in schema") - return - - logger.debug( - "register_discovered_extensions: found %d extension type(s): %s", - len(found), - [info.extension_name for info in found], - ) - for info in found: - # Bottom-up resolve the storage type first, then register the extension - resolved_storage = converter.register_storage_type(info.storage_type) - converter._ensure_extension_type_info( - info.extension_name, - info.extension_metadata, - resolved_storage, - ) - - -def apply_extension_types( - table: "pa.Table", - registry: "LogicalTypeRegistry", # keep registry param for now -) -> "pa.Table": - # (body unchanged — kept exactly as before) - ... -``` - -Keep the `apply_extension_types` and its helpers (`_apply_field`, etc.) exactly as they are — only `register_discovered_extensions` changes. - -Add the old `apply_extension_types` import back: -```python -from orcapod.extension_types.registry import LogicalTypeRegistry -``` - -- [ ] **Step 2: Update `ExtensionAwareDatabase`** - -```python -"""ExtensionAwareDatabase — wrapper that handles extension type registration.""" -from __future__ import annotations - -from collections.abc import Collection, Mapping -from typing import TYPE_CHECKING, Any - -from orcapod.extension_types.database_hooks import ( - apply_extension_types, - register_discovered_extensions, -) -from orcapod.protocols.database_protocols import ArrowDatabaseProtocol - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - -class ExtensionAwareDatabase: - """``ArrowDatabaseProtocol`` wrapper that auto-registers and applies extension types. - - Args: - db: Any ``ArrowDatabaseProtocol`` backend. - converter: The ``UniversalTypeConverter`` to use for extension type - registration and lookup. Callers typically supply - ``data_context.type_converter``. - """ - - def __init__( - self, - db: ArrowDatabaseProtocol, - converter: "UniversalTypeConverter", - ) -> None: - self._db = db - self._converter = converter - - def _process(self, table: "pa.Table | None") -> "pa.Table | None": - """Register extension types and re-wrap columns, or return None unchanged.""" - if table is None: - return None - register_discovered_extensions(self._converter, table.schema) - # apply_extension_types still needs the registry for column re-wrapping - registry = self._converter._logical_type_registry - if registry is not None: - return apply_extension_types(table, registry) - return table - - # All read/write methods delegate exactly as before, replacing self._registry - # usage with self._converter where needed in `at()`: - - def at(self, *path_components: str) -> "ExtensionAwareDatabase": - """Return a scoped view, preserving the extension-aware wrapper.""" - return ExtensionAwareDatabase( - self._db.at(*path_components), - converter=self._converter, - ) - - # ... (all other read/write methods are unchanged from before, just - # delegating self._process(self._db.method(...))) -``` - -Keep all the `get_record_by_id`, `get_all_records`, `add_record`, etc. methods unchanged except that `at()` now passes `converter=self._converter`. - -- [ ] **Step 3: Update call site where `ExtensionAwareDatabase` is constructed** - -Search for all places that construct `ExtensionAwareDatabase`: - -```bash -grep -r "ExtensionAwareDatabase" /home/kurouto/kurouto-jobs/7694626f-534d-48f5-b51f-4bb9c699d932/orcapod-python/src --include="*.py" -l -``` - -For each construction site, change `registry=data_context.logical_type_registry` to `converter=data_context.type_converter`. - -- [ ] **Step 4: Update `test_database_hooks.py`** - -The tests that use `register_discovered_extensions(registry, schema)` need to use `converter`: - -For each test: -1. Create a `UniversalTypeConverter` with the appropriate registry -2. Call `register_discovered_extensions(converter, schema)` instead of `register_discovered_extensions(registry, schema)` - -```python -# Example update pattern in test_database_hooks.py: - -# Before: -# register_discovered_extensions(registry, schema) - -# After: -from orcapod.semantic_types.universal_converter import UniversalTypeConverter -converter = UniversalTypeConverter(logical_type_registry=registry) -register_discovered_extensions(converter, schema) -``` - -- [ ] **Step 5: Run database hook tests** - -```bash -uv run pytest tests/test_extension_types/test_database_hooks.py -v 2>&1 | tail -30 -``` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/database_hooks.py src/orcapod/databases/extension_aware_database.py tests/test_extension_types/test_database_hooks.py -git commit -m "refactor(database_hooks): register_discovered_extensions and ExtensionAwareDatabase now take converter instead of registry" -``` - ---- - -## Task 12: Remove `semantic_registry` from `UniversalTypeConverter`; delete `dataclass_encoding.py`; make `type_utils` private - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Delete: `src/orcapod/semantic_types/dataclass_encoding.py` -- Modify: `src/orcapod/extension_types/type_utils.py` -- Modify: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Remove `semantic_registry` param and usages from `UniversalTypeConverter`** - -In `__init__`, remove `semantic_registry` parameter and `self.semantic_registry = semantic_registry`. - -In `_convert_python_to_arrow`, remove: -```python -# Remove this block: -if self.semantic_registry: - converter = self.semantic_registry.get_converter_for_python_type(python_type) - if converter: - return converter.arrow_struct_type -``` - -In `_convert_arrow_to_python`, remove: -```python -# Remove these blocks: -if self.semantic_registry: - python_type = self.semantic_registry.get_python_type_for_semantic_struct_signature(arrow_type) - if python_type: - return python_type -``` - -In `_create_python_to_arrow_converter`, remove: -```python -# Remove: -if self.semantic_registry: - converter = self.semantic_registry.get_converter_for_python_type(python_type) - if converter: - return converter.python_to_struct_dict -``` - -In `_create_arrow_to_python_converter`, remove: -```python -# Remove: -if self.semantic_registry and pa.types.is_struct(arrow_type): - registered_python_type = ( - self.semantic_registry.get_python_type_for_semantic_struct_signature(arrow_type) - ) - if registered_python_type: - converter = self.semantic_registry.get_converter_for_python_type(registered_python_type) - if converter: - return converter.struct_dict_to_python -``` - -Remove the `from orcapod.semantic_types.semantic_registry import SemanticTypeRegistry` import. - -- [ ] **Step 2: Remove `dataclass_encoding` imports and old dataclass path from converter** - -Remove all imports from `dataclass_encoding`: -```python -# Remove: -from orcapod.semantic_types.dataclass_encoding import ( - DATACLASS_TYPE_FIELD, - _get_type_hints_safe, - dataclass_to_arrow_struct_type, - dataclass_to_struct_dict, - has_dataclass_type_sentinel, - struct_dict_to_dataclass, -) -``` - -In `_convert_python_to_arrow`, remove the dataclass path: -```python -# Remove: -if dataclasses.is_dataclass(python_type) and isinstance(python_type, type): - return dataclass_to_arrow_struct_type(python_type, self) -``` - -In `_convert_arrow_to_python`, remove the dataclass sentinel path: -```python -# Remove the has_dataclass_type_sentinel block (lines referencing has_dataclass_type_sentinel, -# DATACLASS_TYPE_FIELD, struct_dict_to_dataclass, etc.) -``` - -In `_create_python_to_arrow_converter`, remove: -```python -# Remove: -if dataclasses.is_dataclass(python_type) and isinstance(python_type, type): - hints = _get_type_hints_safe(python_type) - field_converters = { - f.name: self.get_python_to_arrow_converter(hints[f.name]) - for f in dataclasses.fields(python_type) - if f.init - } - return lambda obj: dataclass_to_struct_dict(obj, field_converters) -``` - -In `_create_arrow_to_python_converter`, remove: -```python -# Remove the has_dataclass_type_sentinel block -``` - -Remove `import dataclasses` if it's now unused in the converter (check if still needed for the `_create_python_to_arrow_converter` logic after removal). - -- [ ] **Step 3: Delete `dataclass_encoding.py`** - -```bash -rm /home/kurouto/kurouto-jobs/7694626f-534d-48f5-b51f-4bb9c699d932/orcapod-python/src/orcapod/semantic_types/dataclass_encoding.py -git rm src/orcapod/semantic_types/dataclass_encoding.py -``` - -- [ ] **Step 4: Update `type_utils.py` to make `extract_leaf_classes` private** - -```python -# In src/orcapod/extension_types/type_utils.py: -# Rename extract_leaf_classes → _extract_leaf_classes -# Keep the old name as a shim if needed for other callers, or just rename. -``` - -Search for callers: -```bash -grep -r "extract_leaf_classes" /home/kurouto/kurouto-jobs/7694626f-534d-48f5-b51f-4bb9c699d932/orcapod-python/src --include="*.py" -``` - -The only caller was `ensure_types_registered_for_schemas` which we've already replaced with `register_python_class`. Rename the function: - -```python -def _extract_leaf_classes(annotation: Any) -> Iterator[type]: - # (body unchanged) -``` - -Update the module docstring to reflect it's now private. - -- [ ] **Step 5: Update any tests that import `extract_leaf_classes`** - -```bash -grep -r "extract_leaf_classes" /home/kurouto/kurouto-jobs/7694626f-534d-48f5-b51f-4bb9c699d932/orcapod-python/tests --include="*.py" -``` - -Update those tests to use `_extract_leaf_classes` (or remove if the function is no longer tested as part of the public API). - -- [ ] **Step 6: Remove test for `dataclass_encoding.py`** - -Since `dataclass_encoding.py` is deleted, the test file `tests/test_semantic_types/test_dataclass_encoding.py` will fail on import. Remove or archive it: - -```bash -git rm tests/test_semantic_types/test_dataclass_encoding.py -``` - -- [ ] **Step 7: Update `test_universal_converter.py` to not use `semantic_registry`** - -Find all places in `test_universal_converter.py` that pass `semantic_registry=...` to `UniversalTypeConverter(...)` and remove those calls. The tests should pass `logical_type_registry=...` instead (or no argument, using the default context). - -Also update the module-level `python_type_to_arrow_type`, `arrow_type_to_python_type`, `get_conversion_functions` module functions — they call `data_context.type_converter` which no longer uses semantic_registry for type dispatch. Path/UUID types should now go through the logical_type_registry. - -- [ ] **Step 8: Run full test suite** - -```bash -uv run pytest tests/test_semantic_types/ tests/test_extension_types/ -v 2>&1 | tail -40 -``` - -Fix any remaining failures. - -- [ ] **Step 9: Commit** - -```bash -git add -A -git commit -m "refactor(universal_converter): remove semantic_registry usage and dataclass_encoding imports; delete dataclass_encoding.py; make extract_leaf_classes private" -``` - ---- - -## Task 13: Full test suite verification + `extension_types/__init__.py` update - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` -- Verify: entire test suite - -- [ ] **Step 1: Add `DataclassHandlerFactory` and `DataclassLogicalType` to `extension_types/__init__.py`** - -```python -from .dataclass_handler import DataclassHandlerFactory, DataclassLogicalType, DATACLASS_CATEGORY - -__all__ = [ - "LogicalTypeProtocol", - "LogicalTypeFactoryProtocol", - "TypeConverterProtocol", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "make_polars_extension_type", - "ExtensionTypeInfo", - "walk_schema", - "walk_field", - "register_discovered_extensions", - "apply_extension_types", - "DataclassLogicalType", - "DataclassHandlerFactory", - "DATACLASS_CATEGORY", -] -``` - -Update the module docstring to remove the `DataContext.logical_type_registry` access path. - -- [ ] **Step 2: Run the full test suite** - -```bash -uv run pytest tests/ -x 2>&1 | tail -50 -``` - -Fix all failures. Common issues: -- Tests constructing `DataContext` with `logical_type_registry=` → remove that arg -- Tests calling `data_context.logical_type_registry` → use `data_context.type_converter._logical_type_registry` or refactor to use converter methods -- Tests calling `get_default_logical_type_registry()` → use `get_default_context().type_converter._logical_type_registry` or use the converter's registration methods -- Tests calling `factory.create_for_python_type(t)` without `converter=` → add `converter=None` or a stub - -- [ ] **Step 3: Run full test suite and confirm it passes** - -```bash -uv run pytest tests/ 2>&1 | tail -20 -``` -Expected: All tests pass. - -- [ ] **Step 4: Final commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "feat(extension_types): export DataclassHandlerFactory, DataclassLogicalType, DATACLASS_CATEGORY" -``` - ---- - -## Self-Review - -### Spec Coverage Check - -| Spec section | Covered by task | -|---|---| -| `TypeConverterProtocol` added to `extension_types/protocols.py` | Task 1 | -| `LogicalTypeFactoryProtocol`: add `supports_class`, `converter` param | Task 1 | -| `LogicalTypeProtocol`: add `converter` param | Task 1 | -| Built-in types: add `converter` param (accept, ignore) | Task 2 | -| `register_python_class` on converter | Task 3 | -| `register_storage_type` on converter | Task 4 | -| `python_to_storage` / `storage_to_python` on converter | Task 5 | -| Registration pass-throughs | Task 5 | -| Update converter dispatch to pass `converter=self` | Task 5 | -| Simplify `ensure_types_registered_for_schemas` | Task 6 | -| Remove `ensure_*` from registry | Task 6 | -| `DataclassLogicalType` | Task 7 | -| `DataclassHandlerFactory` write path | Task 8 | -| `DataclassHandlerFactory` read path | Task 9 | -| `DataContext.logical_type_registry` removed | Task 10 | -| `get_default_logical_type_registry` removed | Task 10 | -| `v0.1.json` and `context_schema.json` updated | Task 10 | -| `register_discovered_extensions` takes converter | Task 11 | -| `ExtensionAwareDatabase` takes converter | Task 11 | -| Remove `semantic_registry` from converter | Task 12 | -| Delete `dataclass_encoding.py` | Task 12 | -| `extract_leaf_classes` made private | Task 12 | - -All spec requirements are covered. ✓ - -### Known Deviations from Spec - -1. **`register_discovered_extensions`**: The spec proposes simplifying to `for field in schema: converter.register_storage_type(field.type)`. The plan retains `walk_schema` to preserve support for the field-metadata channel (Parquet cold-start where `field.type` is a plain storage type, not a `pa.ExtensionType`). The spec's simplified version only handles in-memory extension types. - -2. **`apply_extension_types`**: Still takes a `LogicalTypeRegistry` argument. `ExtensionAwareDatabase` accesses it via `converter._logical_type_registry`. This is an internal implementation detail. diff --git a/superpowers/plans/2026-06-17-plt-1720-register-python-class-storage-type-cleanup.md b/superpowers/plans/2026-06-17-plt-1720-register-python-class-storage-type-cleanup.md deleted file mode 100644 index 91e767320..000000000 --- a/superpowers/plans/2026-06-17-plt-1720-register-python-class-storage-type-cleanup.md +++ /dev/null @@ -1,831 +0,0 @@ -# PLT-1720: register_python_class storage-type cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `register_python_class` and `register_storage_type` both return storage-safe Arrow types (extension type allowed at top level, no extension types nested inside struct/list fields), delete `_strip_ext_to_storage`, and fix `reconstruct_from_arrow` to register nested types so Parquet round-trips for nested dataclasses work in a fresh process. - -**Architecture:** Three coordinated changes: (1) `_register_python_class_impl` container branches strip any extension type returned by recursive calls before embedding in list/dict; (2) `register_storage_type` strips extension types from struct/list fields when rebuilding the type; (3) `DataclassLogicalTypeFactory.create_for_python_type` replaces the recursive `_strip_ext_to_storage` call with a one-liner, and `reconstruct_from_arrow` adds `converter.register_python_class(annotation)` per field to trigger nested registration. - -**Tech Stack:** Python 3.12+, PyArrow ≥ 20, `uv run pytest` - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/extension_types/protocols.py` | Docstring updates only | -| `src/orcapod/semantic_types/universal_converter.py` | `_register_python_class_impl` container branches; `register_storage_type` struct/list stripping | -| `src/orcapod/extension_types/dataclass_logical_type_factory.py` | Delete `_strip_ext_to_storage`; update `create_for_python_type`; update `reconstruct_from_arrow` | -| `DESIGN_ISSUES.md` | Mark ET1 in progress / update workaround note | -| `tests/test_semantic_types/test_universal_converter.py` | Fix `test_register_storage_type_nested_struct_with_extension` | -| `tests/test_extension_types/test_dataclass_logical_type_factory.py` | New tests: `test_reconstruct_from_arrow_registers_nested_types`, `test_nested_dataclass_parquet_roundtrip` | - ---- - -## Task 1: Update docstrings in protocols.py - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py:27-33` - -- [ ] **Step 1: Update `register_python_class` docstring** - -Replace lines 27–29: -```python - def register_python_class(self, annotation: Any) -> "pa.DataType": - """Traverse a Python annotation and return its Arrow type, registering as needed.""" - ... -``` -With: -```python - def register_python_class(self, annotation: Any) -> "pa.DataType": - """Traverse a Python annotation, register any logical types found, and return - the storage-safe Arrow type. - - The returned type may be a ``pa.ExtensionType`` at the top level for registered - classes (e.g. ``UUID`` → ``orcapod.uuid`` extension type), but struct fields and - list value types at any depth are always plain (non-extension) Arrow types. - - Args: - annotation: A Python type or generic alias (e.g. ``list[str]``, - ``Optional[uuid.UUID]``, a dataclass type). - - Returns: - A storage-safe ``pa.DataType``. May be ``pa.ExtensionType`` at the top level; - never contains nested extension types in struct/list fields. - """ - ... -``` - -- [ ] **Step 2: Update `register_storage_type` docstring** - -Replace lines 31–33: -```python - def register_storage_type(self, arrow_type: "pa.DataType") -> "pa.DataType": - """Traverse an Arrow type bottom-up, registering extension types, and return resolved type.""" - ... -``` -With: -```python - def register_storage_type(self, arrow_type: "pa.DataType") -> "pa.DataType": - """Traverse an Arrow type bottom-up, registering extension types, and return a - storage-safe type. - - The returned type may be a ``pa.ExtensionType`` at the top level, but struct fields - and list value types at any depth are always plain (non-extension) Arrow types. - This invariant makes the return value safe to use as a struct field or list element - type without further stripping. - - Args: - arrow_type: An Arrow type to traverse and register. - - Returns: - A storage-safe ``pa.DataType``. - """ - ... -``` - -- [ ] **Step 3: Run existing protocol tests to confirm no breakage** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -``` -Expected: all PASS - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py -git commit -m "docs(extension-types): update register_python_class and register_storage_type docstrings for storage-safe contract" -``` - ---- - -## Task 2: Fix `register_storage_type` — strip extension types from struct/list fields - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py:441-468` -- Test: `tests/test_semantic_types/test_universal_converter.py` - -The current `register_storage_type` builds new struct/list types with the recursed field types, but does **not** strip an extension type before embedding it in a struct field or list value. Under the storage-safe contract it must strip. - -- [ ] **Step 1: Write the failing test first** - -In `tests/test_semantic_types/test_universal_converter.py`, locate `test_register_storage_type_nested_struct_with_extension` (around line 931). The test currently asserts the extension type is **preserved** in the struct field. Under the new contract it must be **stripped**. Change the last two assertions: - -```python -def test_register_storage_type_nested_struct_with_extension(): - """Extension type nested inside a struct field is stripped to storage type (ET1).""" - import json - import uuid as _u - - ext_name = f"test.nested.{_u.uuid4().hex[:8]}" - category = "test.nested" - metadata = json.dumps({"category": category}).encode() - ArrowExt = make_arrow_extension_type(ext_name, pa.large_string(), metadata=metadata) - PolarsExt = make_polars_extension_type(ext_name, pa.large_string()) - - class _LT: - logical_type_name = ext_name - python_type = str - def get_arrow_extension_type(self): return ArrowExt() - def get_polars_extension_type(self): return PolarsExt() - def python_to_storage(self, v, c=None): return str(v) - def storage_to_python(self, v, c=None): return v - - class _Factory: - def supports_class(self, t): return False - def create_for_python_type(self, t, converter): pass - def reconstruct_from_arrow(self, name, storage_type, meta, converter): - return _LT() - - registry = _make_registry_with_builtins() - registry.register_logical_type_factory(_Factory(), category=category) - converter = _make_converter(registry) - - ext_instance = ArrowExt() - struct_with_ext = pa.struct([pa.field("id", pa.int64()), pa.field("tag", ext_instance)]) - result = converter.register_storage_type(struct_with_ext) - - assert pa.types.is_struct(result) - assert result.field("id").type == pa.int64() - # Storage-safe: extension type inside struct field is stripped to its storage type - assert result.field("tag").type == pa.large_string() - assert not isinstance(result.field("tag").type, pa.ExtensionType) - # Side effect: the extension type IS registered (check via registry) - assert converter._logical_type_registry.get_by_arrow_extension_name(ext_name) is not None -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_register_storage_type_nested_struct_with_extension -v -``` -Expected: FAIL — test currently asserts `isinstance(result.field("tag").type, pa.ExtensionType)`. - -- [ ] **Step 3: Fix `register_storage_type` in `universal_converter.py`** - -Locate the struct branch (around line 443). Replace the struct and list branches: - -**Old struct branch (lines ~443–451):** -```python - # Struct type — recurse into each field, preserving field-level metadata - if pa.types.is_struct(arrow_type): - resolved_fields = [] - for i in range(arrow_type.num_fields): - field = arrow_type.field(i) - resolved_type = self.register_storage_type(field.type) - resolved_fields.append( - pa.field(field.name, resolved_type, nullable=field.nullable, metadata=field.metadata) - ) - return pa.struct(resolved_fields) -``` - -**New struct branch:** -```python - # Struct type — recurse into each field, preserving field-level metadata. - # Strip any extension type from field types before embedding (ET1: Arrow/Polars - # cannot construct arrays whose struct fields are pa.ExtensionType nodes). - if pa.types.is_struct(arrow_type): - resolved_fields = [] - for i in range(arrow_type.num_fields): - field = arrow_type.field(i) - resolved_type = self.register_storage_type(field.type) - if isinstance(resolved_type, pa.ExtensionType): - resolved_type = resolved_type.storage_type # strip: ET1 - resolved_fields.append( - pa.field(field.name, resolved_type, nullable=field.nullable, metadata=field.metadata) - ) - return pa.struct(resolved_fields) -``` - -**Old large_list branch (lines ~453–458):** -```python - # Large list type — preserve value field metadata (used by ARROW:extension:* channel) - if pa.types.is_large_list(arrow_type): - vf = arrow_type.value_field - resolved_value = self.register_storage_type(vf.type) - return pa.large_list( - pa.field(vf.name, resolved_value, nullable=vf.nullable, metadata=vf.metadata) - ) -``` - -**New large_list branch:** -```python - # Large list type — preserve value field metadata (used by ARROW:extension:* channel). - # Strip any extension type from the value type before embedding (ET1). - if pa.types.is_large_list(arrow_type): - vf = arrow_type.value_field - resolved_value = self.register_storage_type(vf.type) - if isinstance(resolved_value, pa.ExtensionType): - resolved_value = resolved_value.storage_type # strip: ET1 - return pa.large_list( - pa.field(vf.name, resolved_value, nullable=vf.nullable, metadata=vf.metadata) - ) -``` - -**Old list branch (lines ~461–466):** -```python - # List type - if pa.types.is_list(arrow_type): - vf = arrow_type.value_field - resolved_value = self.register_storage_type(vf.type) - return pa.list_( - pa.field(vf.name, resolved_value, nullable=vf.nullable, metadata=vf.metadata) - ) -``` - -**New list branch:** -```python - # List type — strip any extension type from the value type (ET1). - if pa.types.is_list(arrow_type): - vf = arrow_type.value_field - resolved_value = self.register_storage_type(vf.type) - if isinstance(resolved_value, pa.ExtensionType): - resolved_value = resolved_value.storage_type # strip: ET1 - return pa.list_( - pa.field(vf.name, resolved_value, nullable=vf.nullable, metadata=vf.metadata) - ) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_register_storage_type_nested_struct_with_extension -v -``` -Expected: PASS - -- [ ] **Step 5: Run the full `register_storage_type` suite** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -k "register_storage_type" -v -``` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "fix(universal-converter): register_storage_type strips extension types from struct/list fields (ET1 storage-safe invariant)" -``` - ---- - -## Task 3: Fix `_register_python_class_impl` container branches — strip before embedding in list/dict - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py:298-326` -- Test: `tests/test_semantic_types/test_universal_converter.py` - -Currently the list/set/dict branches call `self.register_python_class(...)` and embed the result directly in `pa.large_list(...)` or the dict struct. Since `register_python_class` may now return an extension type (e.g. `register_python_class(UUID)` → `orcapod.uuid`), the container branches must strip before embedding to maintain the storage-safe guarantee. - -Note: the registry-hit and factory-dispatch return sites (`return lt.get_arrow_extension_type()`) are **already correct** — they return the extension type directly (top-level extension is allowed). No change needed there. - -- [ ] **Step 1: Verify existing container tests pass before touching anything** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -k "register_python_class_list or register_python_class_dict or register_python_class_set" -v -``` -Expected: PASS - -- [ ] **Step 2: Write a new failing test for `list[UUID]` error behaviour** - -Add at the end of the `register_python_class` block in `tests/test_semantic_types/test_universal_converter.py`: - -```python -def test_register_python_class_list_of_uuid_raises(): - """list[UUID] raises ValueError: UUID is a logical type and cannot be preserved - inside a list value field (ET2 in DESIGN_ISSUES.md). Tracked in PLT-1732.""" - converter = _make_converter() - with pytest.raises(ValueError, match="PLT-1732"): - converter.register_python_class(list[_uuid_module.UUID]) - - -def test_register_python_class_dict_str_uuid_raises(): - """dict[str, UUID] raises ValueError: UUID is a logical type and cannot be preserved - inside a struct field (ET1/ET2 in DESIGN_ISSUES.md). Tracked in PLT-1732.""" - converter = _make_converter() - with pytest.raises(ValueError, match="PLT-1732"): - converter.register_python_class(dict[str, _uuid_module.UUID]) -``` - -- [ ] **Step 3: Run the new tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_register_python_class_list_of_uuid_raises tests/test_semantic_types/test_universal_converter.py::test_register_python_class_dict_str_uuid_raises -v -``` -Expected: FAIL — the list/dict branches currently embed the extension type without raising. - -- [ ] **Step 4: Fix the container branches in `_register_python_class_impl`** - -Locate the list, set, and dict branches (lines ~297–325). Apply stripping after each recursive `register_python_class` call before embedding in a container: - -**Old list branch (lines ~297–304):** -```python - # list[T] → pa.large_list(T) - if origin is list: - if not args: - raise ValueError( - "Unparameterized 'list' is not supported. Use 'list[T]' with a concrete " - "element type (e.g. list[int], list[str])." - ) - return pa.large_list(self.register_python_class(args[0])) -``` - -**New list branch:** -```python - # list[T] → pa.large_list(T). Strip extension type from element (ET1: extension - # types cannot be nested inside list value types). - if origin is list: - if not args: - raise ValueError( - "Unparameterized 'list' is not supported. Use 'list[T]' with a concrete " - "element type (e.g. list[int], list[str])." - ) - inner = self.register_python_class(args[0]) - if isinstance(inner, pa.ExtensionType): - inner = inner.storage_type # strip: ET1 - return pa.large_list(inner) -``` - -**Old set branch (lines ~306–313):** -```python - # set[T] → pa.large_list(T) - if origin is set: - if not args: - raise ValueError( - "Unparameterized 'set' is not supported. Use 'set[T]' with a concrete " - "element type (e.g. set[int], set[str])." - ) - return pa.large_list(self.register_python_class(args[0])) -``` - -**New set branch:** -```python - # set[T] → pa.large_list(T). Strip extension type from element (ET1). - if origin is set: - if not args: - raise ValueError( - "Unparameterized 'set' is not supported. Use 'set[T]' with a concrete " - "element type (e.g. set[int], set[str])." - ) - inner = self.register_python_class(args[0]) - if isinstance(inner, pa.ExtensionType): - inner = inner.storage_type # strip: ET1 - return pa.large_list(inner) -``` - -**Old dict branch (lines ~315–325):** -```python - # dict[K, V] → pa.large_list(struct{key: K, value: V}) - if origin is dict: - if len(args) < 2: - raise ValueError( - "Unparameterized 'dict' is not supported. Use 'dict[K, V]' with concrete " - "key and value types (e.g. dict[str, int])." - ) - key_arrow = self.register_python_class(args[0]) - val_arrow = self.register_python_class(args[1]) - return pa.large_list( - pa.struct([pa.field("key", key_arrow), pa.field("value", val_arrow)]) - ) -``` - -**New dict branch:** -```python - # dict[K, V] → pa.large_list(struct{key: K, value: V}). - # Strip extension types from key and value before embedding in the struct (ET1). - if origin is dict: - if len(args) < 2: - raise ValueError( - "Unparameterized 'dict' is not supported. Use 'dict[K, V]' with concrete " - "key and value types (e.g. dict[str, int])." - ) - key_arrow = self.register_python_class(args[0]) - if isinstance(key_arrow, pa.ExtensionType): - key_arrow = key_arrow.storage_type # strip: ET1 - val_arrow = self.register_python_class(args[1]) - if isinstance(val_arrow, pa.ExtensionType): - val_arrow = val_arrow.storage_type # strip: ET1 - return pa.large_list( - pa.struct([pa.field("key", key_arrow), pa.field("value", val_arrow)]) - ) -``` - -- [ ] **Step 5: Run the new tests to verify they pass** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py::test_register_python_class_list_of_uuid_raises tests/test_semantic_types/test_universal_converter.py::test_register_python_class_dict_str_uuid_raises -v -``` -Expected: PASS - -- [ ] **Step 6: Run the full `register_python_class` suite** - -```bash -uv run pytest tests/test_semantic_types/test_universal_converter.py -k "register_python_class" -v -``` -Expected: all PASS - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/test_universal_converter.py -git commit -m "fix(universal-converter): strip extension types from list/dict container element types in register_python_class (ET1)" -``` - ---- - -## Task 4: Delete `_strip_ext_to_storage` and update `create_for_python_type` - -**Files:** -- Modify: `src/orcapod/extension_types/dataclass_logical_type_factory.py:45-315` -- Test: `tests/test_extension_types/test_dataclass_logical_type_factory.py` - -`_strip_ext_to_storage` (lines 45–90) is now redundant: `register_python_class` already returns a storage-safe type (no nested extension types). The `create_for_python_type` method should replace the recursive `_strip_ext_to_storage(arrow_type)` call with a one-liner strip. - -- [ ] **Step 1: Verify the dataclass factory write-path tests pass before touching anything** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_logical_type_factory.py -v -``` -Expected: all PASS - -- [ ] **Step 2: Delete `_strip_ext_to_storage` and update `create_for_python_type`** - -In `src/orcapod/extension_types/dataclass_logical_type_factory.py`: - -**Delete** the entire `_strip_ext_to_storage` function (lines 45–90, inclusive of docstring). - -**Old block in `create_for_python_type` (lines ~310–315):** -```python - annotation = hints.get(field.name, Any) - arrow_type = converter.register_python_class(annotation) - # Strip extension types from struct field types: pa.array cannot build a - # struct array when a field type is a pa.ExtensionType (see ET1 in - # DESIGN_ISSUES.md). Value conversion is annotation-driven so stripping is safe. - stripped_type = _strip_ext_to_storage(arrow_type) - arrow_fields.append(pa.field(field.name, stripped_type)) -``` - -**New block:** -```python - annotation = hints.get(field.name, Any) - arrow_type = converter.register_python_class(annotation) - # register_python_class returns a storage-safe type: may be extension at the - # top level, but struct fields are always plain. Strip the top-level extension - # type here before inserting into the struct (ET1; see DESIGN_ISSUES.md). - if isinstance(arrow_type, pa.ExtensionType): - arrow_type = arrow_type.storage_type - arrow_fields.append(pa.field(field.name, arrow_type)) -``` - -Also update the comment in `DataclassLogicalType.__init__` (lines ~138–141) that references `_strip_ext_to_storage`: - -**Old:** -```python - # ``storage_type`` is already stripped of nested extension types by - # ``DataclassLogicalTypeFactory.create_for_python_type`` (see ET1 in - # DESIGN_ISSUES.md). ``make_polars_extension_type`` and - # ``pa.array`` both require plain storage types inside structs. -``` - -**New:** -```python - # ``storage_type`` must not contain nested extension types (ET1 in DESIGN_ISSUES.md). - # ``DataclassLogicalTypeFactory.create_for_python_type`` and ``reconstruct_from_arrow`` - # both guarantee this by stripping any top-level extension type from each field's - # Arrow type before inserting it into the struct. -``` - -- [ ] **Step 3: Run the dataclass factory write-path tests to verify they still pass** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_logical_type_factory.py -v -``` -Expected: all PASS - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/extension_types/dataclass_logical_type_factory.py -git commit -m "refactor(dataclass-factory): delete _strip_ext_to_storage, replace with one-liner in create_for_python_type" -``` - ---- - -## Task 5: Fix `reconstruct_from_arrow` — register nested types (read-path completeness fix) - -**Files:** -- Modify: `src/orcapod/extension_types/dataclass_logical_type_factory.py:367-372` -- Test: `tests/test_extension_types/test_dataclass_logical_type_factory.py` - -`reconstruct_from_arrow` currently builds `field_annotations` but never calls `converter.register_python_class` for each annotation. This means nested dataclass types (e.g. `Inner` inside `Outer`) are never registered on the read path, causing `ValueError("Unsupported Python type: Inner.")` in a fresh process. - -- [ ] **Step 1: Write the failing test** - -Add `test_reconstruct_from_arrow_registers_nested_types` to `tests/test_extension_types/test_dataclass_logical_type_factory.py`. This test requires module-level dataclasses. Add them after the existing module-level dataclass definitions (around line 177), before the "DataclassLogicalTypeFactory write-path tests" section: - -```python -@dataclasses.dataclass -class _InnerForRegistrationTest: - """Module-level inner dataclass for registration completeness test.""" - value: int - - -@dataclasses.dataclass -class _OuterForRegistrationTest: - """Module-level outer dataclass for registration completeness test.""" - inner: _InnerForRegistrationTest - label: str -``` - -Then add the test: - -```python -def test_reconstruct_from_arrow_registers_nested_types(): - """reconstruct_from_arrow for Outer must register Inner as a side effect.""" - from orcapod.extension_types.dataclass_logical_type_factory import DataclassLogicalTypeFactory - - # Build the storage type for _OuterForRegistrationTest manually (as it would come - # from Parquet): outer struct with an inner struct field (Inner is stored as a struct, - # NOT as an extension type inside the struct field — that's the ET1 constraint). - inner_storage = pa.struct([pa.field("value", pa.int64())]) - outer_storage = pa.struct([ - pa.field("inner", inner_storage), - pa.field("label", pa.large_string()), - ]) - outer_fqcn = f"{_OuterForRegistrationTest.__module__}.{_OuterForRegistrationTest.__qualname__}" - inner_fqcn = f"{_InnerForRegistrationTest.__module__}.{_InnerForRegistrationTest.__qualname__}" - - factory = DataclassLogicalTypeFactory() - converter = _make_full_converter() - - # Inner is NOT pre-registered - assert converter._logical_type_registry.get_by_python_type(_InnerForRegistrationTest) is None - - # reconstruct_from_arrow for Outer should trigger registration of Inner as a side effect - lt = factory.reconstruct_from_arrow(outer_fqcn, outer_storage, {"category": "orcapod.dataclass"}, converter) - - # Inner must now be registered - assert converter._logical_type_registry.get_by_python_type(_InnerForRegistrationTest) is not None -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -uv run pytest "tests/test_extension_types/test_dataclass_logical_type_factory.py::test_reconstruct_from_arrow_registers_nested_types" -v -``` -Expected: FAIL — `_InnerForRegistrationTest` is not registered after `reconstruct_from_arrow`. - -- [ ] **Step 3: Fix `reconstruct_from_arrow` in `dataclass_logical_type_factory.py`** - -Locate the field-iteration loop inside `reconstruct_from_arrow` (around line 367): - -**Old:** -```python - field_annotations = [] - for field in dataclasses.fields(cls): - if not field.init: - continue - annotation = hints.get(field.name, Any) - field_annotations.append((field.name, annotation)) -``` - -**New:** -```python - field_annotations = [] - for field in dataclasses.fields(cls): - if not field.init: - continue - annotation = hints.get(field.name, Any) - # Register any logical type the field annotation maps to (registration - # completeness invariant: all nested logical types must be registered when - # the outer type is registered). The return value is discarded; only the - # side effect of registration matters here. - converter.register_python_class(annotation) - field_annotations.append((field.name, annotation)) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -uv run pytest "tests/test_extension_types/test_dataclass_logical_type_factory.py::test_reconstruct_from_arrow_registers_nested_types" -v -``` -Expected: PASS - -- [ ] **Step 5: Run the full dataclass factory test suite** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_logical_type_factory.py -v -``` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/dataclass_logical_type_factory.py tests/test_extension_types/test_dataclass_logical_type_factory.py -git commit -m "fix(dataclass-factory): reconstruct_from_arrow registers nested types (registration completeness invariant)" -``` - ---- - -## Task 6: Add Parquet round-trip test for nested dataclasses - -**Files:** -- Test: `tests/test_extension_types/test_dataclass_logical_type_factory.py` - -This test exercises the full fresh-process read path: write a nested dataclass to Parquet, read it back in a converter that has never seen the inner or outer type, call `register_discovered_extensions` + `apply_extension_types`, then convert back to Python. This is the end-to-end regression test for the bug fixed in Task 5. - -The two module-level dataclasses needed (`_InnerForRegistrationTest`, `_OuterForRegistrationTest`) were already added in Task 5. - -- [ ] **Step 1: Write the test** - -Add `test_nested_dataclass_parquet_roundtrip` to `tests/test_extension_types/test_dataclass_logical_type_factory.py`: - -```python -def test_nested_dataclass_parquet_roundtrip(tmp_path): - """Fresh-process Parquet round-trip for a two-level nested dataclass. - - Verifies that register_discovered_extensions triggers the chain: - register_arrow_extension("Outer") → reconstruct_from_arrow - → register_python_class(Inner) → registers Inner - so that storage_to_python can reconstruct the full nested object. - """ - import pyarrow.parquet as pq - from orcapod.extension_types.database_hooks import register_discovered_extensions, apply_extension_types - - # ── Write path ─────────────────────────────────────────────────────────── - write_converter = _make_full_converter() - - inner = _InnerForRegistrationTest(value=42) - outer = _OuterForRegistrationTest(inner=inner, label="hello") - - # Register Outer (which also registers Inner via create_for_python_type) - write_converter.register_python_class(_OuterForRegistrationTest) - - # Serialize to Arrow using python_schema_to_arrow_schema + python_dicts_to_arrow_table - outer_fqcn = f"{_OuterForRegistrationTest.__module__}.{_OuterForRegistrationTest.__qualname__}" - arrow_schema = write_converter.python_schema_to_arrow_schema({"item": _OuterForRegistrationTest}) - rows = [{"item": write_converter.python_to_storage(outer, _OuterForRegistrationTest)}] - table = write_converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - - parquet_path = tmp_path / "nested.parquet" - pq.write_table(table, parquet_path) - - # ── Read path (fresh converter — neither Inner nor Outer pre-registered) ── - read_converter = _make_full_converter() - read_table = pq.read_table(parquet_path) - - # register_discovered_extensions should trigger: Outer → reconstruct_from_arrow - # → register_python_class(Inner) → registers Inner - register_discovered_extensions(read_converter, read_table.schema) - read_table = apply_extension_types(read_table, read_converter._logical_type_registry) - - # Both types must now be registered - assert read_converter._logical_type_registry.get_by_python_type(_OuterForRegistrationTest) is not None - assert read_converter._logical_type_registry.get_by_python_type(_InnerForRegistrationTest) is not None - - # Convert back to Python - rows_out = read_converter.arrow_table_to_python_dicts(read_table) - assert len(rows_out) == 1 - reconstructed = rows_out[0]["item"] - assert isinstance(reconstructed, _OuterForRegistrationTest) - assert isinstance(reconstructed.inner, _InnerForRegistrationTest) - assert reconstructed.inner.value == 42 - assert reconstructed.label == "hello" -``` - -- [ ] **Step 2: Run the test to verify it fails before the fix is in place** - -(This test should already pass since Task 5 fixed `reconstruct_from_arrow`. If running Tasks in order, it will pass. Run it now to confirm.) - -```bash -uv run pytest "tests/test_extension_types/test_dataclass_logical_type_factory.py::test_nested_dataclass_parquet_roundtrip" -v -``` -Expected: PASS (Task 5 already made this possible). - -- [ ] **Step 3: Run the full dataclass factory test suite to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_logical_type_factory.py -v -``` -Expected: all PASS - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_extension_types/test_dataclass_logical_type_factory.py -git commit -m "test(dataclass-factory): add Parquet round-trip test for nested dataclasses" -``` - ---- - -## Task 7: Update `DESIGN_ISSUES.md` — mark ET1 workaround updated - -**Files:** -- Modify: `DESIGN_ISSUES.md` (ET1 entry, around line 1003) - -- [ ] **Step 1: Update ET1** - -Find the ET1 entry. The **Workaround** section currently references `dataclass_handler._strip_ext_to_storage()`. Update it to reflect that `_strip_ext_to_storage` is gone, replaced by the storage-safe contract on `register_python_class` and `register_storage_type`. - -Replace the **Workaround** paragraph in ET1: - -**Old:** -``` -**Workaround:** `dataclass_handler._strip_ext_to_storage()` recursively replaces all -`pa.ExtensionType` nodes with their plain storage types. This stripping is applied in -`DataclassHandlerFactory.create_for_python_type` when building the struct's field types — -so the stored Arrow schema (and thus the struct passed to `make_polars_extension_type` and -`pa.Table.from_pylist`) never contains nested extension types. The consequence is that the -schema for a dataclass extension column reports downgraded inner field types (e.g. -`large_binary` instead of `orcapod.uuid`). This is invisible through the normal conversion -path (all value conversion flows through `converter.storage_to_python`, which is -annotation-driven), but would mislead any code that directly introspects the raw Arrow -or Polars schema of a dataclass extension column's storage fields. - -**Also affects `pa.Table.from_pylist`:** the same restriction applies to PyArrow's -`pa.Table.from_pylist` (and `pa.array`) — neither can build an array from a struct type -whose fields are `pa.ExtensionType` nodes, for the same underlying reason. The stripping -in `create_for_python_type` fixes both issues simultaneously. -``` - -**New:** -``` -**Workaround:** `register_python_class` and `register_storage_type` both uphold a -*storage-safe* invariant: the returned type may be a `pa.ExtensionType` at the top level, -but struct fields and list value types at any depth are always plain (non-extension) types. -`DataclassLogicalTypeFactory.create_for_python_type` strips the top-level extension type -with a one-liner (`if isinstance(arrow_type, pa.ExtensionType): arrow_type = arrow_type.storage_type`) -before inserting it into the struct, so the struct passed to `make_polars_extension_type` -and `pa.Table.from_pylist` never contains nested extension types. The private -`_strip_ext_to_storage` recursive helper was removed in PLT-1720; the stripping is now -trivially correct because the storage-safe invariant guarantees `.storage_type` is always -already clean. - -**Also affects `pa.Table.from_pylist`:** the same restriction applies to PyArrow's -`pa.Table.from_pylist` (and `pa.array`) — neither can build an array from a struct type -whose fields are `pa.ExtensionType` nodes, for the same underlying reason. The stripping -in `create_for_python_type` fixes both issues simultaneously. -``` - -- [ ] **Step 2: Run the full test suite** - -```bash -uv run pytest tests/ -x -q -``` -Expected: all PASS - -- [ ] **Step 3: Commit** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs(design-issues): update ET1 workaround note to reflect removal of _strip_ext_to_storage (PLT-1720)" -``` - ---- - -## Task 8: Final verification and push - -- [ ] **Step 1: Run the complete test suite** - -```bash -uv run pytest tests/ -q -``` -Expected: all PASS, no failures, no errors - -- [ ] **Step 2: Verify the branch is on the right base** - -```bash -git log --oneline extension-type-system..HEAD -``` -Expected: 7 commits (Tasks 1–7), all on top of `extension-type-system`. - -- [ ] **Step 3: Push the branch** - -```bash -git push -u origin eywalker/plt-1720-cleanup-register_python_class-should-return-plain-storage -``` - ---- - -## Self-review checklist - -**Spec coverage:** -- ✅ `register_python_class` container branches strip extension types (Task 3) -- ✅ `register_storage_type` strips extension types from struct/list fields (Task 2) -- ✅ `_strip_ext_to_storage` deleted (Task 4) -- ✅ `create_for_python_type` uses one-liner strip (Task 4) -- ✅ `reconstruct_from_arrow` calls `register_python_class` per field (Task 5) -- ✅ Protocol docstrings updated (Task 1) -- ✅ `DESIGN_ISSUES.md` ET1 updated (Task 7) -- ✅ `test_register_storage_type_nested_struct_with_extension` updated (Task 2) -- ✅ `test_register_python_class_list_of_uuid_raises` added (Task 3) -- ✅ `test_reconstruct_from_arrow_registers_nested_types` added (Task 5) -- ✅ `test_nested_dataclass_parquet_roundtrip` added (Task 6) -- ✅ `database_hooks.py` unchanged (no task needed — already uses `register_storage_type` return value) -- ✅ Existing `register_python_class` tests (`_registry_hit_path`, `_uuid_registry_hit`, `_factory_dispatch`) — these already assert `isinstance(result, pa.ExtensionType)`, which is still correct under the storage-safe contract. No updates needed. - -**Type consistency:** All references to `register_python_class`, `register_storage_type`, `_strip_ext_to_storage`, `create_for_python_type`, and `reconstruct_from_arrow` use the same names as in the source files. - -**No placeholders:** Every step has explicit code or commands. diff --git a/superpowers/plans/2026-06-17-plt-1731-pydantic-logical-type-factory.md b/superpowers/plans/2026-06-17-plt-1731-pydantic-logical-type-factory.md deleted file mode 100644 index 27ff7e605..000000000 --- a/superpowers/plans/2026-06-17-plt-1731-pydantic-logical-type-factory.md +++ /dev/null @@ -1,1380 +0,0 @@ -# PLT-1731 Pydantic Logical Type Factory Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement `PydanticLogicalType` and `PydanticLogicalTypeFactory` for pydantic v2 `BaseModel` subclasses, following the same thin-leaf factory pattern as `DataclassLogicalTypeFactory`. - -**Architecture:** New `pydantic_logical_type_factory.py` mirrors `dataclass_logical_type_factory.py` with no cross-dependency between them. The FQCN walk loop shared by both factories is extracted into `type_utils._walk_fqcn`. Write path delegates all field-type resolution to `converter.register_python_class`; read path delegates to `converter.register_python_class` for the registration completeness invariant. Pydantic is an optional dependency — the factory is importable and `supports_class` returns `False` when pydantic is not installed. - -**Tech Stack:** Python 3.12, PyArrow, Polars, pydantic v2 (`model_fields`, `BaseModel`), `typing.get_type_hints` - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `pyproject.toml` | Modify | Add `pydantic = ["pydantic>=2.0"]` optional extra; add to `all` | -| `src/orcapod/extension_types/type_utils.py` | Modify | Add `_walk_fqcn` shared FQCN walk helper; update module docstring | -| `src/orcapod/extension_types/dataclass_logical_type_factory.py` | Modify | `_import_from_fqcn` delegates to `type_utils._walk_fqcn` | -| `src/orcapod/extension_types/pydantic_logical_type_factory.py` | **Create** | `PYDANTIC_CATEGORY`, `PydanticLogicalType`, `PydanticLogicalTypeFactory`, `_import_pydantic_model_from_fqcn` | -| `src/orcapod/extension_types/__init__.py` | Modify | Export `PYDANTIC_CATEGORY`, `PydanticLogicalType`, `PydanticLogicalTypeFactory` | -| `tests/test_extension_types/test_pydantic_logical_type_factory.py` | **Create** | Full test suite | -| `tests/test_extension_types/test_type_utils.py` | Modify | Add tests for `_walk_fqcn` | - ---- - -## Task 1: Add `pydantic` optional dependency - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add pydantic to optional extras** - -In `pyproject.toml`, find the `[project.optional-dependencies]` section. Add the `pydantic` entry and update `all` to include it: - -```toml -[project.optional-dependencies] -redis = ["redis>=6.2.0"] -ray = ["ray[default]==2.48.0", "ipywidgets>=8.1.7"] -postgresql = ["psycopg[binary]>=3.0"] -spiraldb = [ - "pyspiral>=0.11.0", -] -pydantic = ["pydantic>=2.0"] -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]", "orcapod[pydantic]"] -``` - -- [ ] **Step 2: Install pydantic** - -```bash -uv sync --extra pydantic -``` - -Expected: pydantic installs without errors. - -- [ ] **Step 3: Verify pydantic is available** - -```bash -uv run python -c "import pydantic; print(pydantic.__version__)" -``` - -Expected: prints a version string starting with `2.`. - -- [ ] **Step 4: Commit** - -```bash -git add pyproject.toml -git commit -m "chore(deps): add pydantic>=2.0 as optional dependency" -``` - ---- - -## Task 2: Factor `_walk_fqcn` into `type_utils.py` - -**Files:** -- Modify: `src/orcapod/extension_types/type_utils.py` -- Modify: `src/orcapod/extension_types/dataclass_logical_type_factory.py` -- Modify: `tests/test_extension_types/test_type_utils.py` - -- [ ] **Step 1: Write failing tests for `_walk_fqcn`** - -Add to `tests/test_extension_types/test_type_utils.py`: - -```python -import dataclasses -import pytest - - -# ── _walk_fqcn tests ───────────────────────────────────────────────────────── - -def test_walk_fqcn_resolves_module_level_class(): - """_walk_fqcn resolves a top-level class from its FQCN.""" - from orcapod.extension_types.type_utils import _walk_fqcn - import pathlib - obj = _walk_fqcn("pathlib.Path") - assert obj is pathlib.Path - - -def test_walk_fqcn_resolves_nested_attribute(): - """_walk_fqcn walks nested attribute chains (e.g. module.Outer.Inner).""" - from orcapod.extension_types.type_utils import _walk_fqcn - import os.path - # os.path.join is a function reachable via attribute walk - obj = _walk_fqcn("os.path.join") - assert obj is os.path.join - - -def test_walk_fqcn_raises_import_error_on_bad_module(): - """_walk_fqcn raises ImportError when no module prefix can be imported.""" - from orcapod.extension_types.type_utils import _walk_fqcn - with pytest.raises(ImportError): - _walk_fqcn("nonexistent.module.NoSuchClass") - - -def test_walk_fqcn_raises_import_error_on_missing_attr(): - """_walk_fqcn raises ImportError when module exists but attribute does not.""" - from orcapod.extension_types.type_utils import _walk_fqcn - with pytest.raises(ImportError): - _walk_fqcn("pathlib.NoSuchClass") - - -def test_walk_fqcn_raises_import_error_on_single_part(): - """_walk_fqcn raises ImportError when FQCN has no module separator.""" - from orcapod.extension_types.type_utils import _walk_fqcn - with pytest.raises(ImportError): - _walk_fqcn("justname") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_type_utils.py -k "walk_fqcn" -v -``` - -Expected: all 5 tests FAIL with `ImportError: cannot import name '_walk_fqcn'`. - -- [ ] **Step 3: Add `_walk_fqcn` to `type_utils.py`** - -Replace the full content of `src/orcapod/extension_types/type_utils.py` with: - -```python -"""Utility helpers for Python type annotation inspection and FQCN import. - -Used by the write-side registration trigger to extract leaf Python classes from -complex generic annotations like ``list[dict[A, list[B]]]``, and by logical type -factories to import classes from fully-qualified class names. -""" - -from __future__ import annotations - -import importlib -import typing -from typing import Any, Iterator - - -def _extract_leaf_classes(annotation: Any) -> Iterator[type]: - """Recursively yield all concrete leaf Python classes from a type annotation. - - Unwraps generic aliases (``list[T]``, ``dict[K, V]``, ``Optional[T]``, - ``Union[A, B]``, ``A | B``, etc.) using ``typing.get_origin`` and - ``typing.get_args`` and yields every non-generic leaf found. ``NoneType`` - that appears as a generic argument (from ``Optional`` and - ``Union[..., None]`` / ``T | None``) is skipped — callers see only the - concrete types. When ``type(None)`` is passed directly as the annotation, - it is yielded as-is. - - Non-type, non-generic values (e.g. unresolved string annotations) are - silently skipped. - - Args: - annotation: A Python type or generic alias to inspect. - - Yields: - Concrete Python ``type`` objects found at leaf positions. - - Examples: - >>> list(_extract_leaf_classes(list[int])) - [] - >>> set(_extract_leaf_classes(dict[str, list[MyClass]])) - {, } - """ - origin = typing.get_origin(annotation) - - if origin is None: - # Not a generic alias. Yield only if it is a plain type. - if isinstance(annotation, type): - yield annotation - return - - # Generic alias — recurse into every type argument, skipping NoneType. - for arg in typing.get_args(annotation): - if arg is type(None): - continue - yield from _extract_leaf_classes(arg) - - -def _walk_fqcn(fqcn: str) -> Any: - """Walk a fully-qualified class name and return the resolved object. - - Tries module prefixes from longest to shortest, then walks the remaining - parts as attribute accesses. For example: - - - ``"mypackage.sub.MyClass"`` → import ``mypackage.sub``, then - ``getattr(module, "MyClass")``. - - ``"mypackage.sub.Outer.Inner"`` → import ``mypackage.sub``, then - ``getattr(module, "Outer")``, then ``getattr(Outer, "Inner")``. - - Does **not** validate the type of the resolved object — callers are - responsible for checking that the result is the expected kind of object - (e.g. a dataclass, a ``BaseModel`` subclass). - - Args: - fqcn: Fully-qualified name, e.g. ``"mypackage.sub.MyClass"``. - - Returns: - The resolved Python object. - - Raises: - ImportError: If no valid module+attribute split can be found. - """ - parts = fqcn.split(".") - if len(parts) < 2: - raise ImportError(f"Cannot import from FQCN {fqcn!r}: no module separator found.") - - for i in range(len(parts) - 1, 0, -1): - module_path = ".".join(parts[:i]) - attr_parts = parts[i:] - try: - module = importlib.import_module(module_path) - except (ImportError, ModuleNotFoundError): - continue - obj: Any = module - try: - for attr in attr_parts: - obj = getattr(obj, attr) - except AttributeError: - continue - return obj - - raise ImportError( - f"Cannot import from FQCN {fqcn!r}: no valid module+attribute path found." - ) -``` - -- [ ] **Step 4: Run `_walk_fqcn` tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_type_utils.py -k "walk_fqcn" -v -``` - -Expected: all 5 tests PASS. - -- [ ] **Step 5: Update `_import_from_fqcn` in `dataclass_logical_type_factory.py` to delegate to `_walk_fqcn`** - -Replace the `_import_from_fqcn` function at the bottom of -`src/orcapod/extension_types/dataclass_logical_type_factory.py` with: - -```python -def _import_from_fqcn(fqcn: str) -> type: - """Import a dataclass from its fully-qualified class name. - - Delegates the module-prefix walk to ``type_utils._walk_fqcn``, then - validates the resolved object is a dataclass type. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyClass"``. - - Returns: - The imported dataclass type. - - Raises: - ImportError: If no valid module+attribute split can be found, or if the - resolved object is not a dataclass type. - """ - from orcapod.extension_types.type_utils import _walk_fqcn - - obj: Any = _walk_fqcn(fqcn) - if not dataclasses.is_dataclass(obj) or not isinstance(obj, type): - raise ImportError( - f"{fqcn!r} does not resolve to a dataclass type." - ) - return obj -``` - -Also remove the `import importlib` line at the top of the file since it is no longer used directly. - -- [ ] **Step 6: Run existing dataclass factory tests to verify no regression** - -```bash -uv run pytest tests/test_extension_types/test_dataclass_logical_type_factory.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/extension_types/type_utils.py \ - src/orcapod/extension_types/dataclass_logical_type_factory.py \ - tests/test_extension_types/test_type_utils.py -git commit -m "refactor(type-utils): extract _walk_fqcn shared FQCN helper; delegate from _import_from_fqcn" -``` - ---- - -## Task 3: `PydanticLogicalType` - -**Files:** -- Create: `src/orcapod/extension_types/pydantic_logical_type_factory.py` -- Create: `tests/test_extension_types/test_pydantic_logical_type_factory.py` - -- [ ] **Step 1: Write failing tests for `PydanticLogicalType`** - -Create `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -"""Tests for PydanticLogicalType and PydanticLogicalTypeFactory.""" - -from __future__ import annotations - -import uuid as _uuid_module -from typing import Any - -import pyarrow as pa -import pytest -from pydantic import BaseModel, PrivateAttr - - -# ── Helpers ────────────────────────────────────────────────────────────────── - -class _StubConverter: - """Minimal converter stub for PydanticLogicalType tests.""" - - def python_to_storage(self, value, annotation): - if annotation is str: - return str(value) - if annotation is int: - return int(value) - return value - - def storage_to_python(self, storage_value, annotation): - if annotation is str: - return str(storage_value) - if annotation is int: - return int(storage_value) - return storage_value - - def register_python_class(self, annotation): - if annotation is str: - return pa.large_string() - if annotation is int: - return pa.int64() - raise ValueError(f"No mapping for {annotation}") - - -# ── PydanticLogicalType tests ──────────────────────────────────────────────── - -def test_pydantic_logical_type_is_importable(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - assert PydanticLogicalType is not None - - -def test_pydantic_logical_type_protocol_conformance(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - from orcapod.extension_types.protocols import LogicalTypeProtocol - - class _MyModel(BaseModel): - name: str - count: int - - storage = pa.struct([pa.field("name", pa.large_string()), pa.field("count", pa.int64())]) - lt = PydanticLogicalType( - logical_name="tests._MyModel", - python_type=_MyModel, - storage_type=storage, - field_annotations=[("name", str), ("count", int)], - ) - assert isinstance(lt, LogicalTypeProtocol) - - -def test_pydantic_logical_type_python_to_storage(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - - class _Point(BaseModel): - x: int - y: int - - storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - lt = PydanticLogicalType("tests._Point", _Point, storage, [("x", int), ("y", int)]) - converter = _StubConverter() - - result = lt.python_to_storage(_Point(x=3, y=7), converter) - assert result == {"x": 3, "y": 7} - - -def test_pydantic_logical_type_storage_to_python(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - - class _Point(BaseModel): - x: int - y: int - - storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - lt = PydanticLogicalType("tests._Point2", _Point, storage, [("x", int), ("y", int)]) - converter = _StubConverter() - - result = lt.storage_to_python({"x": 3, "y": 7}, converter) - assert isinstance(result, _Point) - assert result.x == 3 - assert result.y == 7 - - -def test_pydantic_logical_type_logical_type_name(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - - class _Foo(BaseModel): - val: str - - storage = pa.struct([pa.field("val", pa.large_string())]) - lt = PydanticLogicalType("mymod.Foo", _Foo, storage, [("val", str)]) - assert lt.logical_type_name == "mymod.Foo" - - -def test_pydantic_logical_type_python_type(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - - class _Bar(BaseModel): - val: str - - storage = pa.struct([pa.field("val", pa.large_string())]) - lt = PydanticLogicalType("mymod.Bar", _Bar, storage, [("val", str)]) - assert lt.python_type is _Bar - - -def test_python_to_storage_raises_when_converter_none(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - - class _DC(BaseModel): - x: int - - storage = pa.struct([pa.field("x", pa.int64())]) - lt = PydanticLogicalType("mymod._DC", _DC, storage, [("x", int)]) - with pytest.raises(ValueError, match="converter"): - lt.python_to_storage(_DC(x=1), None) - - -def test_storage_to_python_raises_when_converter_none(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalType - - class _DC2(BaseModel): - x: int - - storage = pa.struct([pa.field("x", pa.int64())]) - lt = PydanticLogicalType("mymod._DC2", _DC2, storage, [("x", int)]) - with pytest.raises(ValueError, match="converter"): - lt.storage_to_python({"x": 1}, None) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v -``` - -Expected: all tests FAIL with `ModuleNotFoundError: No module named 'orcapod.extension_types.pydantic_logical_type_factory'`. - -- [ ] **Step 3: Create `pydantic_logical_type_factory.py` with `PydanticLogicalType`** - -Create `src/orcapod/extension_types/pydantic_logical_type_factory.py`: - -```python -"""PydanticLogicalType and PydanticLogicalTypeFactory. - -Provides the ``PydanticLogicalType`` logical type implementation and the -``PydanticLogicalTypeFactory`` that synthesises and reconstructs -``PydanticLogicalType`` instances for pydantic v2 ``BaseModel`` subclasses. - -Write path (``create_for_python_type``): - Iterates model fields via ``model_fields`` (pydantic v2 API), delegates - field Arrow-type resolution to the converter via ``register_python_class``, - and returns a ``PydanticLogicalType`` backed by a ``pa.struct`` extension - type. - -Read path (``reconstruct_from_arrow``): - Imports the model by fully-qualified class name, resolves field annotations - against the (already bottom-up resolved) storage type, and returns a - ``PydanticLogicalType``. - -Category tag: ``"orcapod.pydantic"`` -""" - -from __future__ import annotations - -import json -import logging -from typing import TYPE_CHECKING, Any - -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type -from orcapod.extension_types.type_utils import _walk_fqcn -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from orcapod.extension_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) - -#: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. -PYDANTIC_CATEGORY = "orcapod.pydantic" - - -class PydanticLogicalType: - """Logical type binding a pydantic ``BaseModel`` subclass to its Arrow extension type. - - Stores the model's fully-qualified class name as the Arrow extension name - and a ``pa.struct`` of the model fields as the storage type. - - No Arrow-type reasoning lives here — all field-type resolution is owned by - the converter and completed before this object is constructed. - - Args: - logical_name: Fully-qualified class name (e.g. ``"mymodule.sub.MyModel"``). - Used as both the logical type name and the Arrow extension name. - python_type: The pydantic ``BaseModel`` subclass. - storage_type: The Arrow ``pa.StructType`` for the model fields. - field_annotations: Ordered list of ``(field_name, python_annotation)`` - pairs matching the fields in ``storage_type``. - - Example: - >>> lt = PydanticLogicalType( - ... "mymod.Point", Point, - ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), - ... [("x", int), ("y", int)], - ... ) - >>> lt.python_to_storage(Point(x=1, y=2), converter) - {"x": 1, "y": 2} - """ - - def __init__( - self, - logical_name: str, - python_type: type, - storage_type: pa.StructType, - field_annotations: list[tuple[str, Any]], - ) -> None: - self._logical_name = logical_name - self._python_type = python_type - self._storage_type = storage_type - self._field_annotations = field_annotations - - _metadata = json.dumps({"category": PYDANTIC_CATEGORY}).encode("utf-8") - self._arrow_ext_class = make_arrow_extension_type( - logical_name, storage_type, metadata=_metadata - ) - self._arrow_ext: pa.ExtensionType | None = None - # ``storage_type`` must not contain nested extension types (ET1 in DESIGN_ISSUES.md). - # On the write path, ``PydanticLogicalTypeFactory.create_for_python_type`` strips any - # top-level extension type from each field's Arrow type before inserting it into the - # struct. On the read path, ``reconstruct_from_arrow`` receives a ``storage_type`` - # already guaranteed storage-safe by ``register_storage_type``. - self._polars_ext_class = make_polars_extension_type(logical_name, storage_type) - self._polars_ext: pl.BaseExtension | None = None - - @property - def logical_type_name(self) -> str: - """Fully-qualified class name used as the logical type identifier.""" - return self._logical_name - - @property - def python_type(self) -> type: - """The pydantic ``BaseModel`` subclass this logical type represents.""" - return self._python_type - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for this model. - - Returns: - A cached ``pa.ExtensionType`` instance with ``extension_name`` equal to - the fully-qualified class name and ``storage_type`` equal to the struct - of the model fields. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for this model. - - Returns: - A cached ``pl.BaseExtension`` instance. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext - - def python_to_storage(self, value: Any, converter: TypeConverterProtocol | None) -> dict[str, Any]: - """Convert a pydantic model instance to an Arrow-compatible struct dict. - - Iterates ``_field_annotations`` and delegates each field's conversion to - ``converter.python_to_storage``. - - Args: - value: A pydantic model instance of type ``python_type``. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dict mapping field names to their Arrow storage values. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "PydanticLogicalType.python_to_storage requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - return { - name: converter.python_to_storage(getattr(value, name), annotation) - for name, annotation in self._field_annotations - } - - def storage_to_python(self, storage_value: Any, converter: TypeConverterProtocol | None) -> Any: - """Reconstruct a pydantic model instance from an Arrow struct dict. - - Args: - storage_value: A dict mapping field names to Arrow storage values. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A pydantic model instance of type ``python_type``. Pydantic validation - runs on construction, ensuring the model is always in a valid state. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "PydanticLogicalType.storage_to_python requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - kwargs = { - name: converter.storage_to_python(storage_value[name], annotation) - for name, annotation in self._field_annotations - } - return self._python_type(**kwargs) -``` - -- [ ] **Step 4: Run `PydanticLogicalType` tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - -k "not factory" -v -``` - -Expected: all 8 `PydanticLogicalType` tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/pydantic_logical_type_factory.py \ - tests/test_extension_types/test_pydantic_logical_type_factory.py -git commit -m "feat(pydantic-factory): add PydanticLogicalType" -``` - ---- - -## Task 4: `PydanticLogicalTypeFactory` — write path - -**Files:** -- Modify: `src/orcapod/extension_types/pydantic_logical_type_factory.py` -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` - -- [ ] **Step 1: Add module-level models and write-path tests to the test file** - -Append to `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -# ── Module-level models for factory tests ──────────────────────────────────── -# Must be at module scope (not inside functions) so FQCN reconstruction works. - -class _FlatModel(BaseModel): - name: str - count: int - - -class _ModelWithUUID(BaseModel): - id: _uuid_module.UUID - label: str - - -class _ModelWithList(BaseModel): - tags: list[str] - count: int - - -class _ModelWithDict(BaseModel): - meta: dict[str, int] - - -class _InnerModel(BaseModel): - value: int - - -class _OuterModel(BaseModel): - inner: _InnerModel - label: str - - -class _ModelWithPrivateAttr(BaseModel): - name: str - _cache: str = PrivateAttr(default="") - - -# ── Factory helper ──────────────────────────────────────────────────────────── - -def _make_full_converter(): - """Make a UniversalTypeConverter with builtin types + PydanticLogicalTypeFactory.""" - from pydantic import BaseModel as _BaseModel - from orcapod.extension_types.builtin_logical_types import LogicalPath, LogicalUUID, LogicalUPath - from orcapod.extension_types.registry import LogicalTypeRegistry - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory, PYDANTIC_CATEGORY - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - registry = LogicalTypeRegistry(logical_types=[LogicalPath(), LogicalUUID(), LogicalUPath()]) - factory = PydanticLogicalTypeFactory() - registry.register_logical_type_factory(factory, category=PYDANTIC_CATEGORY, python_bases=[_BaseModel]) - return UniversalTypeConverter(logical_type_registry=registry) - - -# ── PydanticLogicalTypeFactory write-path tests ─────────────────────────────── - -def test_factory_supports_class_pydantic_model(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - assert factory.supports_class(_FlatModel) is True - - -def test_factory_supports_class_non_pydantic(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - import dataclasses - - @dataclasses.dataclass - class _DC: - x: int - - factory = PydanticLogicalTypeFactory() - assert factory.supports_class(str) is False - assert factory.supports_class(int) is False - assert factory.supports_class(_DC) is False - - -def test_factory_create_flat_model(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory, PydanticLogicalType - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_FlatModel, converter=converter) - - assert isinstance(lt, PydanticLogicalType) - storage = lt.get_arrow_extension_type().storage_type - assert pa.types.is_struct(storage) - assert storage.field("name").type == pa.large_string() - assert storage.field("count").type == pa.int64() - - -def test_factory_create_model_with_uuid_field(): - """UUID field → plain storage type (large_binary) in the struct, not extension type (ET1).""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_ModelWithUUID, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - id_field_type = storage.field("id").type - assert id_field_type == pa.large_binary() - assert not isinstance(id_field_type, pa.ExtensionType) - - -def test_factory_create_model_with_list_field(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_ModelWithList, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - assert pa.types.is_large_list(storage.field("tags").type) - assert storage.field("tags").type.value_type == pa.large_string() - - -def test_factory_create_model_with_dict_field(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_ModelWithDict, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - meta_type = storage.field("meta").type - assert pa.types.is_large_list(meta_type) - assert pa.types.is_struct(meta_type.value_type) - field_names = {meta_type.value_type.field(i).name for i in range(meta_type.value_type.num_fields)} - assert field_names == {"key", "value"} - - -def test_factory_rejects_local_class(): - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - def _make_local(): - class _Local(BaseModel): - x: int - return _Local - - LocalModel = _make_local() - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - with pytest.raises(ValueError, match="local"): - factory.create_for_python_type(LocalModel, converter=converter) - - -def test_private_fields_not_stored(): - """Private attributes (PrivateAttr) must not appear in the Arrow struct.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_ModelWithPrivateAttr, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - field_names = {storage.field(i).name for i in range(storage.num_fields)} - assert "name" in field_names - assert "_cache" not in field_names - assert storage.num_fields == 1 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - -k "factory" -v 2>&1 | head -30 -``` - -Expected: all factory tests FAIL with `ImportError: cannot import name 'PydanticLogicalTypeFactory'`. - -- [ ] **Step 3: Add `PydanticLogicalTypeFactory` and `_import_pydantic_model_from_fqcn` to the module** - -Append to `src/orcapod/extension_types/pydantic_logical_type_factory.py`: - -```python - -class PydanticLogicalTypeFactory: - """Stateless factory that synthesises and reconstructs ``PydanticLogicalType`` instances. - - **Write path** (``create_for_python_type``): derives Arrow struct type from the - model fields by delegating to ``converter.register_python_class`` per field. - Only fields in ``model_fields`` are stored — computed fields and private - attributes are excluded. - - **Read path** (``reconstruct_from_arrow``): imports the model by FQCN, matches - fields against the already-resolved ``storage_type``, and returns a - ``PydanticLogicalType``. - - Category tag: ``"orcapod.pydantic"`` - - Register with:: - - from pydantic import BaseModel - converter.register_logical_type_factory( - PydanticLogicalTypeFactory(), - category="orcapod.pydantic", - python_bases=[BaseModel], - ) - - Example: - >>> factory = PydanticLogicalTypeFactory() - >>> factory.supports_class(MyModel) - True - >>> factory.supports_class(str) - False - """ - - def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a pydantic ``BaseModel`` subclass. - - Args: - python_type: Any Python type. - - Returns: - True if pydantic is installed and ``python_type`` is a ``BaseModel`` - subclass. False if pydantic is not installed. - """ - try: - from pydantic import BaseModel - except ImportError: - return False - return isinstance(python_type, type) and issubclass(python_type, BaseModel) - - def create_for_python_type( - self, - python_type: type, - converter: TypeConverterProtocol, - ) -> PydanticLogicalType: - """Synthesise a ``PydanticLogicalType`` for a pydantic model (write path). - - Derives the FQCN, obtains type hints, and resolves each field's Arrow type - via ``converter.register_python_class``. Only fields present in - ``model_fields`` are stored — computed fields and private attributes are - excluded. Rejects local / unnamed classes. - - Args: - python_type: A pydantic ``BaseModel`` subclass. - converter: The active converter for field-type resolution. - - Returns: - A ``PydanticLogicalType`` ready for registration. - - Raises: - ValueError: If ``python_type`` is a local class (``__qualname__`` contains - ``""``). - """ - import typing - - fqcn = f"{python_type.__module__}.{python_type.__qualname__}" - if "" in fqcn: - raise ValueError( - f"Cannot register local class {python_type!r} as a PydanticLogicalType — " - f"local classes have no stable fully-qualified class name and cannot be " - f"reconstructed on read. Define the model at module level." - ) - - try: - hints = typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {python_type!r}: {exc}" - ) from exc - - arrow_fields = [] - field_annotations = [] - for field_name in python_type.model_fields: - annotation = hints.get(field_name, Any) - arrow_type = converter.register_python_class(annotation) - # Strip top-level extension type before inserting into the struct (ET1; - # see DESIGN_ISSUES.md): Arrow cannot represent extension types inside - # struct field types. - if isinstance(arrow_type, pa.ExtensionType): - arrow_type = arrow_type.storage_type - arrow_fields.append(pa.field(field_name, arrow_type)) - field_annotations.append((field_name, annotation)) - - storage_type = pa.struct(arrow_fields) - logger.debug("PydanticLogicalTypeFactory: synthesised %r for %r", fqcn, python_type) - return PydanticLogicalType(fqcn, python_type, storage_type, field_annotations) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict[str, Any], - converter: TypeConverterProtocol, - ) -> PydanticLogicalType: - """Reconstruct a ``PydanticLogicalType`` from Arrow schema metadata (read path). - - Imports the model from its FQCN (``arrow_extension_name``), then matches - the model field annotations against the fields in ``storage_type``. - ``storage_type`` is already bottom-up resolved by ``register_storage_type`` - before this method is called. - - Args: - arrow_extension_name: FQCN of the pydantic model (Arrow extension name). - storage_type: Already-resolved ``pa.StructType`` for the model fields. - metadata: Full parsed metadata JSON dict (always contains ``"category"``). - converter: The active converter (used for registration completeness invariant). - - Returns: - A ``PydanticLogicalType`` ready for registration. - - Raises: - ImportError: If the class cannot be imported from ``arrow_extension_name``. - ValueError: If ``storage_type`` is not a struct type. - """ - import typing - - if not pa.types.is_struct(storage_type): - raise ValueError( - f"PydanticLogicalTypeFactory.reconstruct_from_arrow: expected a struct " - f"storage type for {arrow_extension_name!r}, got {storage_type!r}." - ) - - cls = _import_pydantic_model_from_fqcn(arrow_extension_name) - - try: - hints = typing.get_type_hints(cls) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {cls!r}: {exc}" - ) from exc - - field_annotations = [] - for field_name in cls.model_fields: - annotation = hints.get(field_name, Any) - # Register any logical type the field annotation maps to (registration - # completeness invariant: all nested logical types must be registered when - # the outer type is registered). The return value is discarded. - converter.register_python_class(annotation) - field_annotations.append((field_name, annotation)) - - logger.debug( - "PydanticLogicalTypeFactory: reconstructed %r from Arrow", arrow_extension_name - ) - return PydanticLogicalType( - arrow_extension_name, cls, storage_type, field_annotations - ) - - -def _import_pydantic_model_from_fqcn(fqcn: str) -> type: - """Import a pydantic ``BaseModel`` subclass from its fully-qualified class name. - - Delegates the module-prefix walk to ``type_utils._walk_fqcn``, then - validates the resolved object is a ``BaseModel`` subclass. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyModel"``. - - Returns: - The imported ``BaseModel`` subclass. - - Raises: - ImportError: If no valid module+attribute split can be found, or if the - resolved object is not a ``BaseModel`` subclass. - """ - from pydantic import BaseModel - - obj: Any = _walk_fqcn(fqcn) - if not (isinstance(obj, type) and issubclass(obj, BaseModel)): - raise ImportError( - f"{fqcn!r} does not resolve to a pydantic BaseModel subclass." - ) - return obj -``` - -- [ ] **Step 4: Run write-path tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - -k "factory" -v -``` - -Expected: all write-path factory tests PASS (reconstruct tests will still fail — that's fine for now). - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/pydantic_logical_type_factory.py \ - tests/test_extension_types/test_pydantic_logical_type_factory.py -git commit -m "feat(pydantic-factory): add PydanticLogicalTypeFactory write path" -``` - ---- - -## Task 5: Read path, round-trip tests, and Parquet integration - -**Files:** -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` - -The `reconstruct_from_arrow` implementation is already in place from Task 4. This task adds the remaining tests that exercise the read path, value round-trips, and Parquet end-to-end. - -- [ ] **Step 1: Add module-level models for read-path and round-trip tests** - -Append to `tests/test_extension_types/test_pydantic_logical_type_factory.py` (after the write-path tests): - -```python -# ── Module-level models for read-path and round-trip tests ─────────────────── - -class _RoundTripPoint(BaseModel): - x: int - y: int - - -class _RoundTripRecord(BaseModel): - record_id: _uuid_module.UUID - label: str -``` - -- [ ] **Step 2: Add read-path and round-trip tests** - -Append to `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -# ── PydanticLogicalTypeFactory read-path tests ──────────────────────────────── - -def test_factory_reconstruct_from_arrow(): - """reconstruct_from_arrow rebuilds the logical type from the Arrow struct.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory, PydanticLogicalType - - storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - metadata = {"category": "orcapod.pydantic"} - fqcn = f"{_RoundTripPoint.__module__}.{_RoundTripPoint.__qualname__}" - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.reconstruct_from_arrow(fqcn, storage, metadata, converter=converter) - - assert isinstance(lt, PydanticLogicalType) - assert lt.python_type is _RoundTripPoint - assert lt.logical_type_name == fqcn - - -def test_factory_reconstruct_from_arrow_invalid_fqcn(): - """ImportError if the FQCN cannot be resolved.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - storage = pa.struct([pa.field("x", pa.int64())]) - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - - with pytest.raises(ImportError): - factory.reconstruct_from_arrow( - "nonexistent.module.NoSuchModel", storage, {"category": "orcapod.pydantic"}, converter - ) - - -def test_reconstruct_from_arrow_registers_nested_types(): - """reconstruct_from_arrow for Outer must register Inner as a side effect.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - inner_storage = pa.struct([pa.field("value", pa.int64())]) - outer_storage = pa.struct([ - pa.field("inner", inner_storage), - pa.field("label", pa.large_string()), - ]) - outer_fqcn = f"{_OuterModel.__module__}.{_OuterModel.__qualname__}" - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - - # Inner is NOT pre-registered - assert converter._logical_type_registry.get_by_python_type(_InnerModel) is None - - factory.reconstruct_from_arrow(outer_fqcn, outer_storage, {"category": "orcapod.pydantic"}, converter) - - # Inner must now be registered as a side effect - assert converter._logical_type_registry.get_by_python_type(_InnerModel) is not None - - -# ── Value round-trip tests ──────────────────────────────────────────────────── - -def test_pydantic_python_to_storage_round_trip(): - """python_to_storage → storage_to_python returns an equivalent model.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - converter = _make_full_converter() - factory = PydanticLogicalTypeFactory() - lt = factory.create_for_python_type(_RoundTripPoint, converter=converter) - converter.register_logical_type(lt) - - point = _RoundTripPoint(x=10, y=20) - storage_value = lt.python_to_storage(point, converter) - assert storage_value == {"x": 10, "y": 20} - - reconstructed = lt.storage_to_python(storage_value, converter) - assert isinstance(reconstructed, _RoundTripPoint) - assert reconstructed.x == 10 - assert reconstructed.y == 20 - - -def test_pydantic_with_uuid_round_trip(): - """Round-trip a pydantic model with a UUID field.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - converter = _make_full_converter() - factory = PydanticLogicalTypeFactory() - lt = factory.create_for_python_type(_RoundTripRecord, converter=converter) - converter.register_logical_type(lt) - - u = _uuid_module.UUID("12345678-1234-5678-1234-567812345678") - record = _RoundTripRecord(record_id=u, label="hello") - - storage_value = lt.python_to_storage(record, converter) - assert storage_value["label"] == "hello" - assert storage_value["record_id"] == u.bytes - - reconstructed = lt.storage_to_python(storage_value, converter) - assert isinstance(reconstructed, _RoundTripRecord) - assert reconstructed.record_id == u - assert reconstructed.label == "hello" - - -# ── Parquet integration test ────────────────────────────────────────────────── - -def test_nested_pydantic_model_parquet_roundtrip(tmp_path): - """Fresh-process Parquet round-trip for a two-level nested pydantic model. - - Verifies that register_discovered_extensions triggers the chain: - register_arrow_extension("Outer") -> reconstruct_from_arrow - -> register_python_class(Inner) -> registers Inner - so that storage_to_python can reconstruct the full nested object. - """ - import pyarrow.parquet as pq - from orcapod.extension_types.database_hooks import register_discovered_extensions, apply_extension_types - - # ── Write path ─────────────────────────────────────────────────────────── - write_converter = _make_full_converter() - - inner = _InnerModel(value=42) - outer = _OuterModel(inner=inner, label="hello") - - write_converter.register_python_class(_OuterModel) - - arrow_schema = write_converter.python_schema_to_arrow_schema({"item": _OuterModel}) - rows = [{"item": outer}] - table = write_converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - - parquet_path = tmp_path / "nested_pydantic.parquet" - pq.write_table(table, parquet_path) - - # ── Read path (fresh converter — neither Inner nor Outer pre-registered) ── - read_converter = _make_full_converter() - read_table = pq.read_table(parquet_path) - - register_discovered_extensions(read_converter, read_table.schema) - read_table = apply_extension_types(read_table, read_converter._logical_type_registry) - - assert read_converter._logical_type_registry.get_by_python_type(_OuterModel) is not None - assert read_converter._logical_type_registry.get_by_python_type(_InnerModel) is not None - - rows_out = read_converter.arrow_table_to_python_dicts(read_table) - assert len(rows_out) == 1 - reconstructed = rows_out[0]["item"] - assert isinstance(reconstructed, _OuterModel) - assert isinstance(reconstructed.inner, _InnerModel) - assert reconstructed.inner.value == 42 - assert reconstructed.label == "hello" -``` - -- [ ] **Step 3: Run all tests for the new factory** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_extension_types/test_pydantic_logical_type_factory.py -git commit -m "test(pydantic-factory): add read-path, round-trip, and Parquet integration tests" -``` - ---- - -## Task 6: Export from `__init__.py` and full test suite - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` - -- [ ] **Step 1: Add exports to `__init__.py`** - -In `src/orcapod/extension_types/__init__.py`, add the pydantic import and update `__all__`: - -```python -from .pydantic_logical_type_factory import PYDANTIC_CATEGORY, PydanticLogicalType, PydanticLogicalTypeFactory -``` - -Add to `__all__`: - -```python - # PLT-1731 - "PYDANTIC_CATEGORY", - "PydanticLogicalType", - "PydanticLogicalTypeFactory", -``` - -The full updated `__init__.py` should be: - -```python -"""Arrow/Polars extension type system for orcapod. - -This subpackage provides the registry and protocol for logical types that map -between Python objects and their Arrow/Polars extension type representation. - -Built-in registrations (``LogicalPath``, ``LogicalUPath``, ``LogicalUUID``) are -wired into ``DataContext`` via ``contexts/data/v0.1.json``. Use -``get_default_context().type_converter.register_python_class()`` to register new -types, ``register_logical_type_factory()`` to add factories, and -``apply_extension_types()`` to re-wrap Arrow tables with their registered extension types. - -``DataclassLogicalTypeFactory`` provides automatic registration for Python dataclasses: -register it with a ``LogicalTypeRegistry`` and any dataclass used in a ``FunctionPod`` -will be auto-registered on pod declaration. - -``PydanticLogicalTypeFactory`` provides automatic registration for pydantic v2 -``BaseModel`` subclasses: register it with a ``LogicalTypeRegistry`` using -``python_bases=[BaseModel]`` and any model used in a ``FunctionPod`` will be -auto-registered on pod declaration. Requires the ``pydantic`` optional extra. -""" - -from __future__ import annotations - -from .protocols import LogicalTypeProtocol, LogicalTypeFactoryProtocol -from .registry import LogicalTypeRegistry, make_arrow_extension_type, make_polars_extension_type -from .schema_walker import ExtensionTypeInfo, walk_field, walk_schema -from .database_hooks import apply_extension_types, register_discovered_extensions -from .dataclass_logical_type_factory import DATACLASS_CATEGORY, DataclassLogicalType, DataclassLogicalTypeFactory -from .pydantic_logical_type_factory import PYDANTIC_CATEGORY, PydanticLogicalType, PydanticLogicalTypeFactory - -__all__ = [ - "LogicalTypeProtocol", - "LogicalTypeFactoryProtocol", - "LogicalTypeRegistry", - "make_arrow_extension_type", - "make_polars_extension_type", - # PLT-1654 - "ExtensionTypeInfo", - "walk_schema", - "walk_field", - # PLT-1655 - "register_discovered_extensions", - "apply_extension_types", - # PLT-1705 - "DATACLASS_CATEGORY", - "DataclassLogicalType", - "DataclassLogicalTypeFactory", - # PLT-1731 - "PYDANTIC_CATEGORY", - "PydanticLogicalType", - "PydanticLogicalTypeFactory", -] -``` - -- [ ] **Step 2: Verify the exports are importable** - -```bash -uv run python -c " -from orcapod.extension_types import ( - PYDANTIC_CATEGORY, PydanticLogicalType, PydanticLogicalTypeFactory -) -print('PYDANTIC_CATEGORY:', PYDANTIC_CATEGORY) -print('PydanticLogicalType:', PydanticLogicalType) -print('PydanticLogicalTypeFactory:', PydanticLogicalTypeFactory) -" -``` - -Expected output: -``` -PYDANTIC_CATEGORY: orcapod.pydantic -PydanticLogicalType: -PydanticLogicalTypeFactory: -``` - -- [ ] **Step 3: Run the full extension_types test suite** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all tests PASS with no regressions. - -- [ ] **Step 4: Run the full test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "feat(pydantic-factory): export PydanticLogicalType and PydanticLogicalTypeFactory from extension_types" -``` diff --git a/superpowers/plans/2026-06-18-plt-1701-wire-factories-into-default-registry.md b/superpowers/plans/2026-06-18-plt-1701-wire-factories-into-default-registry.md deleted file mode 100644 index 334cb4e35..000000000 --- a/superpowers/plans/2026-06-18-plt-1701-wire-factories-into-default-registry.md +++ /dev/null @@ -1,571 +0,0 @@ -# PLT-1701: Wire Factories into Default LogicalTypeRegistry Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Wire `DataclassLogicalTypeFactory` and `PydanticLogicalTypeFactory` into the default `LogicalTypeRegistry` so dataclass- and pydantic-annotated pod fields are handled automatically with zero user-side setup. - -**Architecture:** Four targeted changes: (1) promote pydantic to a required dep, (2) harden `PydanticLogicalTypeFactory.supports_class` by dropping the `try/except ImportError` guard, (3) add a `factories` constructor parameter to `LogicalTypeRegistry` that calls `register_logical_type_factory` for each entry, (4) wire both factories into `v0.1.json`. A new test file verifies registry construction and default-context end-to-end behaviour. - -**Tech Stack:** Python 3.11+, PyArrow, pydantic v2, `uv` for dependency management. - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `pyproject.toml` | Modify | Move pydantic from optional to required dependency | -| `src/orcapod/extension_types/pydantic_logical_type_factory.py` | Modify | `supports_class`: drop `try/except ImportError`, import pydantic directly | -| `src/orcapod/extension_types/registry.py` | Modify | `LogicalTypeRegistry.__init__`: add `factories` parameter | -| `src/orcapod/contexts/data/v0.1.json` | Modify | Add `factories` list under `logical_type_registry._config` | -| `tests/test_extension_types/test_default_context_factories.py` | Create | Registry unit tests + default-context integration tests | - ---- - -## Task 0: Create and check out the feature branch - -**Files:** (none — git only) - -- [ ] **Step 1: Create and check out the branch from `extension-type-system`** - -```bash -git checkout extension-type-system -git checkout -b eywalker/plt-1701-wire-dataclasshandlerfactory-into-the-default -git branch --show-current -``` - -Expected: prints `eywalker/plt-1701-wire-dataclasshandlerfactory-into-the-default`. - ---- - -## Task 1: Promote pydantic to a required dependency - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Move pydantic into `[project.dependencies]`** - -In `pyproject.toml`, add `"pydantic>=2.0"` to `[project.dependencies]` and remove the `pydantic` entry from `[project.optional-dependencies]` (keep the `all` extra but remove `"orcapod[pydantic]"` from it): - -```toml -[project] -dependencies = [ - "xxhash", - "networkx", - "typing_extensions", - "matplotlib>=3.10.3", - "pandas>=2.2.3", - "pyyaml>=6.0.2", - "pyarrow>=20.0.0", - "polars>=1.36.0", - "beartype>=0.21.0", - "deltalake>=1.0.2", - "graphviz>=0.21", - "gitpython>=3.1.45", - "universal-pathlib>=0.3.8", - "starfix>=0.2.0", - "pygraphviz>=1.14", - "tzdata>=2024.1", - "uuid-utils>=0.11.1", - "s3fs>=2025.12.0", - "pymongo>=4.15.5", - "basedpyright>=1.38.1", - "pydantic>=2.0", -] - -[project.optional-dependencies] -redis = ["redis>=6.2.0"] -ray = ["ray[default]==2.48.0", "ipywidgets>=8.1.7"] -postgresql = ["psycopg[binary]>=3.0"] -spiraldb = [ - "pyspiral>=0.11.0", -] -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]"] -``` - -- [ ] **Step 2: Re-sync the environment** - -```bash -uv sync -``` - -Expected: pydantic is resolved as a required dep. No errors. - -- [ ] **Step 3: Verify pydantic is available** - -```bash -uv run python -c "import pydantic; print(pydantic.__version__)" -``` - -Expected: prints a version string starting with `2.`. - -- [ ] **Step 4: Run the existing pydantic factory tests to confirm nothing broke** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add pyproject.toml -git commit -m "chore(deps): promote pydantic to required dependency" -``` - ---- - -## Task 2: Harden `PydanticLogicalTypeFactory.supports_class` - -**Files:** -- Modify: `src/orcapod/extension_types/pydantic_logical_type_factory.py:211-225` - -The current `supports_class` wraps its pydantic import in a `try/except ImportError` that silently returns `False` when pydantic is absent. Now that pydantic is required, this guard is dead code and should be removed. The behaviour when pydantic IS installed is identical — no new failing test is needed; the existing `test_pydantic_logical_type_factory.py` suite covers it. - -- [ ] **Step 1: Update `supports_class` in `pydantic_logical_type_factory.py`** - -Replace the current `supports_class` method (lines ~211–225): - -```python -def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a pydantic ``BaseModel`` subclass. - - Args: - python_type: Any Python type. - - Returns: - True if ``python_type`` is a ``BaseModel`` subclass. - """ - from pydantic import BaseModel - return isinstance(python_type, type) and issubclass(python_type, BaseModel) -``` - -- [ ] **Step 2: Verify pydantic tests still pass** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/extension_types/pydantic_logical_type_factory.py -git commit -m "fix(pydantic-factory): drop try/except in supports_class — pydantic is now required" -``` - ---- - -## Task 3: Add `factories` parameter to `LogicalTypeRegistry.__init__` - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py:205-213` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_extension_types/test_default_context_factories.py` with just the registry unit tests: - -```python -"""Tests for LogicalTypeRegistry factories parameter and default context factory wiring.""" - -from __future__ import annotations - -import dataclasses - -import pytest - -from orcapod.extension_types.dataclass_logical_type_factory import ( - DataclassLogicalTypeFactory, - DATACLASS_CATEGORY, -) -from orcapod.extension_types.pydantic_logical_type_factory import ( - PydanticLogicalTypeFactory, - PYDANTIC_CATEGORY, -) -from orcapod.extension_types.registry import LogicalTypeRegistry - - -# ── Module-level dataclasses (local classes cannot be registered) ──────────── - -@dataclasses.dataclass -class _SimplePoint: - x: int - y: int - - -# ── Registry constructor unit tests ───────────────────────────────────────── - -def test_registry_factories_param_registers_category(): - """factories param registers the factory under the given category.""" - factory = DataclassLogicalTypeFactory() - registry = LogicalTypeRegistry( - factories=[{"factory": factory, "category": DATACLASS_CATEGORY, "python_bases": [object]}] - ) - assert registry._category_factories.get(DATACLASS_CATEGORY) is factory - - -def test_registry_factories_param_registers_python_base(): - """factories param registers the factory under each python_base.""" - factory = DataclassLogicalTypeFactory() - registry = LogicalTypeRegistry( - factories=[{"factory": factory, "category": DATACLASS_CATEGORY, "python_bases": [object]}] - ) - assert registry._python_class_factories.get(object) is factory - - -def test_registry_factories_param_empty_list_is_noop(): - """factories=[] constructs successfully with no registered factories.""" - registry = LogicalTypeRegistry(factories=[]) - assert registry._category_factories == {} - assert registry._python_class_factories == {} - - -def test_registry_factories_param_none_is_noop(): - """factories=None (default) constructs successfully.""" - registry = LogicalTypeRegistry(factories=None) - assert registry._category_factories == {} -``` - -- [ ] **Step 2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_extension_types/test_default_context_factories.py::test_registry_factories_param_registers_category -v -``` - -Expected: `FAILED` — `LogicalTypeRegistry.__init__` does not yet accept `factories`. - -- [ ] **Step 3: Update `LogicalTypeRegistry.__init__` in `registry.py`** - -Replace the current `__init__` signature and body (lines ~205–212): - -```python -def __init__( - self, - logical_types: list[LogicalTypeProtocol] | None = None, - factories: list[dict] | None = None, -) -> None: - self._by_logical_name: dict[str, LogicalTypeProtocol] = {} - self._by_arrow_name: dict[str, LogicalTypeProtocol] = {} - self._by_python_type: dict[type, LogicalTypeProtocol] = {} - self._category_factories: dict[str, LogicalTypeFactoryProtocol] = {} - self._python_class_factories: dict[type, LogicalTypeFactoryProtocol] = {} - for lt in (logical_types or []): - self.register_logical_type(lt) - for entry in (factories or []): - self.register_logical_type_factory( - entry["factory"], - category=entry.get("category"), - python_bases=entry.get("python_bases", []), - ) -``` - -- [ ] **Step 4: Run the registry unit tests** - -```bash -uv run pytest tests/test_extension_types/test_default_context_factories.py::test_registry_factories_param_registers_category tests/test_extension_types/test_default_context_factories.py::test_registry_factories_param_registers_python_base tests/test_extension_types/test_default_context_factories.py::test_registry_factories_param_empty_list_is_noop tests/test_extension_types/test_default_context_factories.py::test_registry_factories_param_none_is_noop -v -``` - -Expected: all 4 tests pass. - -- [ ] **Step 5: Run the existing registry tests to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/registry.py tests/test_extension_types/test_default_context_factories.py -git commit -m "feat(registry): add factories parameter to LogicalTypeRegistry.__init__" -``` - ---- - -## Task 4: Wire both factories into `v0.1.json` - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Write the failing default-context tests** - -Append these tests to `tests/test_extension_types/test_default_context_factories.py`: - -```python -# ── Default context integration tests ──────────────────────────────────────── -# -# All tests use create_registry().get_context() — NOT get_default_context() — -# to avoid cross-test contamination via the global singleton cache. - -from orcapod.contexts import create_registry - - -def test_default_context_has_dataclass_factory(): - """Default context registers DataclassLogicalTypeFactory under orcapod.dataclass.""" - ctx = create_registry().get_context() - registry = ctx.type_converter._logical_type_registry - factory = registry._category_factories.get(DATACLASS_CATEGORY) - assert isinstance(factory, DataclassLogicalTypeFactory) - - -def test_default_context_has_pydantic_factory(): - """Default context registers PydanticLogicalTypeFactory under orcapod.pydantic.""" - ctx = create_registry().get_context() - registry = ctx.type_converter._logical_type_registry - factory = registry._category_factories.get(PYDANTIC_CATEGORY) - assert isinstance(factory, PydanticLogicalTypeFactory) -``` - -- [ ] **Step 2: Run those two tests to confirm they fail** - -```bash -uv run pytest tests/test_extension_types/test_default_context_factories.py::test_default_context_has_dataclass_factory tests/test_extension_types/test_default_context_factories.py::test_default_context_has_pydantic_factory -v -``` - -Expected: both `FAILED` — factories not yet in `v0.1.json`. - -- [ ] **Step 3: Add the `factories` list to `v0.1.json`** - -In `src/orcapod/contexts/data/v0.1.json`, find the `logical_type_registry` object spec -(under `type_converter._config`) and add `"factories"` alongside `"logical_types"`: - -```json -"logical_type_registry": { - "_class": "orcapod.extension_types.registry.LogicalTypeRegistry", - "_config": { - "logical_types": [ - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUUID", - "_config": {} - } - ], - "factories": [ - { - "factory": { - "_class": "orcapod.extension_types.dataclass_logical_type_factory.DataclassLogicalTypeFactory", - "_config": {} - }, - "category": "orcapod.dataclass", - "python_bases": [{"_type": "builtins.object"}] - }, - { - "factory": { - "_class": "orcapod.extension_types.pydantic_logical_type_factory.PydanticLogicalTypeFactory", - "_config": {} - }, - "category": "orcapod.pydantic", - "python_bases": [{"_type": "pydantic.BaseModel"}] - } - ] - } -} -``` - -`{"_type": "builtins.object"}` resolves to the `object` class via `parse_objectspec`. -`{"_type": "pydantic.BaseModel"}` resolves to `pydantic.BaseModel` the same way — no -instance is created, the class itself is passed as a `python_bases` entry. - -- [ ] **Step 4: Run the default-context factory tests** - -```bash -uv run pytest tests/test_extension_types/test_default_context_factories.py::test_default_context_has_dataclass_factory tests/test_extension_types/test_default_context_factories.py::test_default_context_has_pydantic_factory -v -``` - -Expected: both pass. - -- [ ] **Step 5: Verify the existing context tests still pass** - -```bash -uv run pytest test-objective/unit/test_contexts.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json tests/test_extension_types/test_default_context_factories.py -git commit -m "feat(contexts): wire DataclassLogicalTypeFactory and PydanticLogicalTypeFactory into v0.1 default context" -``` - ---- - -## Task 5: Add end-to-end integration tests via the default context - -**Files:** -- Modify: `tests/test_extension_types/test_default_context_factories.py` - -These tests prove that a user can define a dataclass or pydantic model and use it immediately as a pod field type via the default context — no manual factory registration. - -- [ ] **Step 1: Add module-level pydantic model to the test file** - -At the top of `tests/test_extension_types/test_default_context_factories.py`, after the existing module-level dataclass, add: - -```python -from pydantic import BaseModel - - -class _SimpleModel(BaseModel): - name: str - score: float -``` - -- [ ] **Step 2: Add the auto-registration tests** - -Append to `tests/test_extension_types/test_default_context_factories.py`: - -```python -import pyarrow as pa -from orcapod.extension_types.database_hooks import apply_extension_types, register_discovered_extensions - - -def test_default_context_dataclass_auto_registered_on_use(): - """register_python_class on a dataclass works zero-setup via the default context.""" - converter = create_registry().get_context().type_converter - arrow_type = converter.register_python_class(_SimplePoint) - assert isinstance(arrow_type, pa.ExtensionType) - fqcn = f"{_SimplePoint.__module__}.{_SimplePoint.__qualname__}" - assert arrow_type.extension_name == fqcn - - -def test_default_context_pydantic_auto_registered_on_use(): - """register_python_class on a pydantic model works zero-setup via the default context.""" - converter = create_registry().get_context().type_converter - arrow_type = converter.register_python_class(_SimpleModel) - assert isinstance(arrow_type, pa.ExtensionType) - fqcn = f"{_SimpleModel.__module__}.{_SimpleModel.__qualname__}" - assert arrow_type.extension_name == fqcn -``` - -- [ ] **Step 3: Add the Parquet round-trip tests** - -Append to `tests/test_extension_types/test_default_context_factories.py`: - -```python -import pyarrow.parquet as pq - - -def test_default_context_dataclass_parquet_roundtrip(tmp_path): - """Dataclass round-trips through Parquet with no manual factory registration.""" - # Write path — fresh context, no manual factory setup - write_converter = create_registry().get_context().type_converter - arrow_schema = write_converter.python_schema_to_arrow_schema({"point": _SimplePoint}) - rows = [{"point": _SimplePoint(x=3, y=7)}] - table = write_converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - - parquet_path = tmp_path / "point.parquet" - pq.write_table(table, parquet_path) - - # Read path — another fresh context, no manual factory setup - read_converter = create_registry().get_context().type_converter - read_table = pq.read_table(parquet_path) - register_discovered_extensions(read_converter, read_table.schema) - read_table = apply_extension_types(read_table, read_converter._logical_type_registry) - - rows_out = read_converter.arrow_table_to_python_dicts(read_table) - assert len(rows_out) == 1 - result = rows_out[0]["point"] - assert isinstance(result, _SimplePoint) - assert result.x == 3 - assert result.y == 7 - - -def test_default_context_pydantic_parquet_roundtrip(tmp_path): - """Pydantic model round-trips through Parquet with no manual factory registration.""" - # Write path — fresh context, no manual factory setup - write_converter = create_registry().get_context().type_converter - arrow_schema = write_converter.python_schema_to_arrow_schema({"model": _SimpleModel}) - rows = [{"model": _SimpleModel(name="alice", score=9.5)}] - table = write_converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - - parquet_path = tmp_path / "model.parquet" - pq.write_table(table, parquet_path) - - # Read path — another fresh context, no manual factory setup - read_converter = create_registry().get_context().type_converter - read_table = pq.read_table(parquet_path) - register_discovered_extensions(read_converter, read_table.schema) - read_table = apply_extension_types(read_table, read_converter._logical_type_registry) - - rows_out = read_converter.arrow_table_to_python_dicts(read_table) - assert len(rows_out) == 1 - result = rows_out[0]["model"] - assert isinstance(result, _SimpleModel) - assert result.name == "alice" - assert result.score == 9.5 -``` - -- [ ] **Step 4: Run all tests in the new test file** - -```bash -uv run pytest tests/test_extension_types/test_default_context_factories.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Run the full extension_types test suite to check for regressions** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all tests pass (the existing xfail on `test_list_of_nested_dataclass_parquet_roundtrip` still xfails as expected). - -- [ ] **Step 6: Commit** - -```bash -git add tests/test_extension_types/test_default_context_factories.py -git commit -m "test(registry): add default context factory registration and Parquet round-trip tests" -``` - ---- - -## Task 6: Final verification and PR - -- [ ] **Step 1: Run the complete test suite** - -```bash -uv run pytest tests/ test-objective/ -v -``` - -Expected: all tests pass. No new failures. - -- [ ] **Step 2: Create the PR** - -```bash -gh pr create \ - --base extension-type-system \ - --title "feat(registry): wire DataclassLogicalTypeFactory and PydanticLogicalTypeFactory into default context (PLT-1701)" \ - --body "$(cat <<'EOF' -## Summary - -- Promotes pydantic to a required dependency (was optional extra) -- Adds `factories` parameter to `LogicalTypeRegistry.__init__` — accepts a list of dicts with `factory`, `category`, and `python_bases` keys; each entry is registered via `register_logical_type_factory` at construction time -- Drops `try/except ImportError` guard in `PydanticLogicalTypeFactory.supports_class` — pydantic is now always available -- Wires `DataclassLogicalTypeFactory` and `PydanticLogicalTypeFactory` into `v0.1.json` under `logical_type_registry._config.factories`; uses `{"_type": "..."}` object-specs for `python_bases` so `parse_objectspec` resolves them to actual type objects -- Adds integration tests verifying zero-setup dataclass/pydantic auto-registration and Parquet round-trips via the default context - -## Test plan - -- [ ] `uv run pytest tests/test_extension_types/ -v` — all pass -- [ ] `uv run pytest test-objective/unit/test_contexts.py -v` — all pass -- [ ] `uv run pytest tests/ test-objective/ -v` — full suite passes - -Closes PLT-1701 -EOF -)" -``` - -Expected: PR URL printed. Verify it targets `extension-type-system`, not `main`. diff --git a/superpowers/plans/2026-06-21-plt-1737-schema-metadata-cleaner.md b/superpowers/plans/2026-06-21-plt-1737-schema-metadata-cleaner.md deleted file mode 100644 index 7d951f0d4..000000000 --- a/superpowers/plans/2026-06-21-plt-1737-schema-metadata-cleaner.md +++ /dev/null @@ -1,969 +0,0 @@ -# Schema Metadata Cleaner + starfix v0.3.0 Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement `clean_schema_for_hashing` and `has_extension_metadata` utilities in a new `schema_cleaner` module, wire them into `StarfixArrowHasher`, and bump the `starfix` dependency to `~=0.3.0`. - -**Architecture:** A new pure-function module (`src/orcapod/hashing/schema_cleaner.py`) strips non-`ARROW:extension:*` metadata from Arrow schemas recursively. `StarfixArrowHasher.hash_schema` and `hash_table` call the cleaner, then pass `include_metadata=True/False` to `ArrowDigester` depending on whether any extension metadata survived cleaning. Schemas with no extension metadata hash byte-for-byte identically to pre-v0.3.0 output (stability invariant). - -**Tech Stack:** Python 3.12, PyArrow 20+, starfix 0.3.x, pytest, uv - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `pyproject.toml` | Modify | Fix `starfix` specifier to `~=0.3.0` | -| `src/orcapod/hashing/schema_cleaner.py` | **Create** | `clean_schema_for_hashing`, `has_extension_metadata`, private helpers | -| `src/orcapod/hashing/arrow_hashers.py` | Modify | Wire cleaner into `StarfixArrowHasher.hash_schema` and `hash_table` | -| `tests/test_hashing/test_schema_cleaner.py` | **Create** | Unit tests for the cleaner module | -| `tests/test_hashing/test_starfix_arrow_hasher.py` | Modify | Add `TestHashSchemaExtensionAware` and `TestHashTableExtensionAware` | -| `CHANGELOG.md` | Modify | Add entry for v0.3.0 adoption + hash-invalidation caveat | - ---- - -## Task 1: Fix dependency specifier in pyproject.toml - -**Files:** -- Modify: `pyproject.toml` - -The `uv add` command wrote `starfix>=0.3.0,<0.4.0`. The spec calls for `~=0.3.0` (PEP 440 compatible release — equivalent, more idiomatic). - -- [ ] **Step 1: Edit pyproject.toml** - -In `pyproject.toml`, change the starfix line: - -```toml -# Before: -"starfix>=0.3.0,<0.4.0", - -# After: -"starfix~=0.3.0", -``` - -- [ ] **Step 2: Sync lockfile** - -```bash -uv lock -``` - -Expected: `uv.lock` regenerated or unchanged (both `>=0.3.0,<0.4.0` and `~=0.3.0` resolve to 0.3.1 — the only 0.3.x release). - -- [ ] **Step 3: Verify starfix version installed** - -```bash -uv run python -c "import importlib.metadata; print(importlib.metadata.version('starfix'))" -``` - -Expected output: -``` -0.3.1 -``` - -- [ ] **Step 4: Commit** - -```bash -git add pyproject.toml uv.lock -git commit -m "chore(deps): pin starfix to ~=0.3.0 (PLT-1737)" -``` - ---- - -## Task 2: Implement `schema_cleaner.py` — cleaner (TDD) - -**Files:** -- Create: `tests/test_hashing/test_schema_cleaner.py` -- Create: `src/orcapod/hashing/schema_cleaner.py` - -### Background - -Arrow schemas carry two kinds of metadata: -- **Schema-level metadata**: `schema.metadata` — a `dict[bytes, bytes] | None` -- **Field-level metadata**: `field.metadata` — same type, on each `pa.Field` - -Nested types (`struct`, `list`, `large_list`, `fixed_size_list`, `map`) have child fields, each of which can also carry metadata. The cleaner must recurse. - -Only keys starting with `b"ARROW:extension:"` are identity-bearing. Everything else is stripped. - -PyArrow API notes relevant to this task: -- `pa.types.is_struct(t)` / `pa.types.is_list(t)` / `pa.types.is_large_list(t)` / `pa.types.is_fixed_size_list(t)` / `pa.types.is_map(t)` — type predicates -- `struct_type.field(i)` — returns the i-th child field (0-indexed) -- `list_type.value_field` / `large_list_type.value_field` / `fixed_size_list_type.value_field` — value field -- `fixed_size_list_type.list_size` — integer size -- `map_type.key_field` / `map_type.item_field` — key and item fields -- `map_type.keys_sorted` — bool -- `pa.list_(value_field)` — creates a variable-length list type -- `pa.list_(value_field, list_size)` — creates a fixed-size list type -- `pa.large_list(value_field)` — creates a large variable-length list type -- `pa.map_(key_field, item_field, keys_sorted)` — creates a map type (accepts `pa.Field` objects) -- `pa.field(name, type, nullable=True, metadata=None)` — creates a field -- `pa.schema(fields, metadata=None)` — creates a schema - -- [ ] **Step 1: Write failing tests for `clean_schema_for_hashing`** - -Create `tests/test_hashing/test_schema_cleaner.py`: - -```python -""" -Tests for the schema metadata cleaner. - -Coverage --------- -- clean_schema_for_hashing: extension-free, extension-only, mixed metadata, - schema-level metadata, nested types (struct, list, large_list, fixed_size_list, - map), deep nesting (list>, struct>), fixture snapshots -- has_extension_metadata: False for no metadata, True for extension field, recurse - into nested types, True for deep extension-only field, False on cleaned schema -""" - -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.hashing.schema_cleaner import clean_schema_for_hashing, has_extension_metadata - -_EXT_NAME = b"ARROW:extension:name" -_EXT_META = b"ARROW:extension:metadata" -_COMMENT = b"comment" -_VENDOR = b"vendor:tag" - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _ext_meta(name: str) -> dict[bytes, bytes]: - return {_EXT_NAME: name.encode()} - -def _mixed_meta(name: str) -> dict[bytes, bytes]: - return {_EXT_NAME: name.encode(), _COMMENT: b"ignore me", _VENDOR: b"v1"} - - -# --------------------------------------------------------------------------- -# TestCleanSchemaForHashing -# --------------------------------------------------------------------------- - -class TestCleanSchemaForHashing: - def test_extension_free_schema_metadata_stripped_to_empty(self): - """Schema-level metadata with no ARROW:extension:* keys is dropped entirely.""" - schema = pa.schema( - [pa.field("x", pa.int32())], - metadata={_COMMENT: b"hi", _VENDOR: b"v1"}, - ) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.metadata in (None, {}) - - def test_extension_free_field_metadata_stripped_to_empty(self): - """Field metadata with no ARROW:extension:* keys is dropped.""" - schema = pa.schema([ - pa.field("x", pa.int32(), metadata={_COMMENT: b"hi"}), - ]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.field("x").metadata in (None, {}) - - def test_extension_only_schema_is_noop(self): - """Schema with only ARROW:extension:* metadata is unchanged by the cleaner.""" - meta = {_EXT_NAME: b"my.type", _EXT_META: b"{}"} - schema = pa.schema( - [pa.field("x", pa.int32(), metadata=meta)], - metadata={_EXT_NAME: b"schema.ext"}, - ) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.field("x").metadata == meta - assert cleaned.metadata == {_EXT_NAME: b"schema.ext"} - - def test_mixed_metadata_only_extension_keys_survive(self): - """Mixed metadata: only ARROW:extension:* keys survive on the field.""" - schema = pa.schema([ - pa.field("x", pa.int32(), metadata=_mixed_meta("my.type")), - ]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.field("x").metadata == {_EXT_NAME: b"my.type"} - - def test_schema_level_mixed_metadata_filtered(self): - """Schema-level mixed metadata: only ARROW:extension:* keys survive.""" - schema = pa.schema( - [pa.field("x", pa.int32())], - metadata={_EXT_NAME: b"s.ext", _COMMENT: b"drop me"}, - ) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.metadata == {_EXT_NAME: b"s.ext"} - - def test_names_types_nullability_preserved(self): - """Cleaner never touches names, types, or nullability.""" - schema = pa.schema([ - pa.field("a", pa.int32(), nullable=False, metadata={_COMMENT: b"x"}), - pa.field("b", pa.float64(), nullable=True, metadata={_EXT_NAME: b"t"}), - ]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.field("a").name == "a" - assert cleaned.field("a").type == pa.int32() - assert cleaned.field("a").nullable is False - assert cleaned.field("b").name == "b" - assert cleaned.field("b").type == pa.float64() - assert cleaned.field("b").nullable is True - - def test_returns_new_schema_object(self): - """clean_schema_for_hashing always returns a new schema, never mutates.""" - schema = pa.schema([pa.field("x", pa.int32(), metadata={_COMMENT: b"hi"})]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned is not schema - # Original is untouched - assert schema.field("x").metadata == {_COMMENT: b"hi"} - - -# --------------------------------------------------------------------------- -# TestCleanFieldRecursion -# --------------------------------------------------------------------------- - -class TestCleanFieldRecursion: - def test_struct_child_metadata_cleaned(self): - """Struct child fields have their non-extension metadata stripped.""" - schema = pa.schema([ - pa.field("s", pa.struct([ - pa.field("child", pa.int32(), metadata=_mixed_meta("child.t")), - ])), - ]) - cleaned = clean_schema_for_hashing(schema) - child = cleaned.field("s").type.field(0) - assert child.metadata == {_EXT_NAME: b"child.t"} - - def test_list_value_field_metadata_cleaned(self): - """list value field metadata is filtered.""" - schema = pa.schema([ - pa.field("lst", pa.list_( - pa.field("item", pa.int32(), metadata=_mixed_meta("item.t")) - )), - ]) - cleaned = clean_schema_for_hashing(schema) - value_field = cleaned.field("lst").type.value_field - assert value_field.metadata == {_EXT_NAME: b"item.t"} - - def test_large_list_value_field_metadata_cleaned(self): - """large_list value field metadata is filtered.""" - schema = pa.schema([ - pa.field("lst", pa.large_list( - pa.field("item", pa.int32(), metadata=_mixed_meta("item.t")) - )), - ]) - cleaned = clean_schema_for_hashing(schema) - value_field = cleaned.field("lst").type.value_field - assert value_field.metadata == {_EXT_NAME: b"item.t"} - - def test_fixed_size_list_value_field_metadata_cleaned(self): - """fixed_size_list value field metadata is filtered; list_size preserved.""" - schema = pa.schema([ - pa.field("lst", pa.list_( - pa.field("item", pa.int32(), metadata=_mixed_meta("item.t")), - 3, - )), - ]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.field("lst").type.list_size == 3 - value_field = cleaned.field("lst").type.value_field - assert value_field.metadata == {_EXT_NAME: b"item.t"} - - def test_map_key_field_metadata_cleaned(self): - """map key field metadata is filtered.""" - schema = pa.schema([ - pa.field("m", pa.map_( - pa.field("key", pa.string(), nullable=False, metadata={_COMMENT: b"drop"}), - pa.int32(), - )), - ]) - cleaned = clean_schema_for_hashing(schema) - key_field = cleaned.field("m").type.key_field - assert key_field.metadata in (None, {}) - - def test_map_item_field_metadata_cleaned(self): - """map item field metadata is filtered.""" - schema = pa.schema([ - pa.field("m", pa.map_( - pa.string(), - pa.field("value", pa.int32(), metadata=_mixed_meta("val.t")), - )), - ]) - cleaned = clean_schema_for_hashing(schema) - item_field = cleaned.field("m").type.item_field - assert item_field.metadata == {_EXT_NAME: b"val.t"} - - def test_deep_list_of_struct_cleaned(self): - """list>: extension metadata on the deeply-nested struct - child field is preserved; unrelated keys are stripped at every level.""" - inner_field = pa.field("x", pa.int32(), metadata=_mixed_meta("inner.t")) - schema = pa.schema([ - pa.field("lst", pa.list_( - pa.field("item", pa.struct([inner_field]), - metadata={_COMMENT: b"mid level drop"}), - )), - ]) - cleaned = clean_schema_for_hashing(schema) - item_field = cleaned.field("lst").type.value_field - # Mid-level (value_field) metadata stripped - assert item_field.metadata in (None, {}) - # Deep child preserved only for extension keys - deep_child = item_field.type.field(0) - assert deep_child.metadata == {_EXT_NAME: b"inner.t"} - - def test_deep_struct_of_struct_cleaned(self): - """struct>: extension metadata on the grandchild field - is preserved; unrelated keys at every level are stripped.""" - grandchild = pa.field("gc", pa.int32(), metadata=_mixed_meta("gc.t")) - schema = pa.schema([ - pa.field("outer", pa.struct([ - pa.field("inner", pa.struct([grandchild]), - metadata={_COMMENT: b"drop"}), - ])), - ]) - cleaned = clean_schema_for_hashing(schema) - inner_field = cleaned.field("outer").type.field(0) - assert inner_field.metadata in (None, {}) - gc = inner_field.type.field(0) - assert gc.metadata == {_EXT_NAME: b"gc.t"} - - -# --------------------------------------------------------------------------- -# TestCleanSchemaFixtures (input → expected snapshot pairs) -# --------------------------------------------------------------------------- - -class TestCleanSchemaFixtures: - def test_fixture_no_metadata(self): - """Schema with no metadata at all: cleaner returns schema without metadata.""" - schema = pa.schema([ - pa.field("id", pa.int32()), - pa.field("val", pa.float64()), - ]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.metadata in (None, {}) - assert cleaned.field("id").metadata in (None, {}) - assert cleaned.field("val").metadata in (None, {}) - - def test_fixture_extension_field(self): - """Schema with extension field: snapshot the cleaned (identity) output.""" - schema = pa.schema([ - pa.field("t", pa.binary(), metadata={ - _EXT_NAME: b"my_pkg.MyType", - _EXT_META: b'{"version":1}', - }), - ]) - cleaned = clean_schema_for_hashing(schema) - assert cleaned.field("t").metadata == { - _EXT_NAME: b"my_pkg.MyType", - _EXT_META: b'{"version":1}', - } - - def test_fixture_mixed_deep(self): - """Schema with mixed metadata at two levels of nesting.""" - schema = pa.schema([ - pa.field("top", pa.int32(), metadata={_COMMENT: b"drop", _EXT_NAME: b"top.t"}), - pa.field("nested", pa.struct([ - pa.field("child", pa.utf8(), metadata={ - _COMMENT: b"also drop", - _EXT_NAME: b"child.t", - }), - ]), metadata={_VENDOR: b"remove"}), - ], metadata={_COMMENT: b"schema comment", _EXT_NAME: b"schema.ext"}) - - cleaned = clean_schema_for_hashing(schema) - - assert cleaned.metadata == {_EXT_NAME: b"schema.ext"} - assert cleaned.field("top").metadata == {_EXT_NAME: b"top.t"} - assert cleaned.field("nested").metadata in (None, {}) - assert cleaned.field("nested").type.field(0).metadata == {_EXT_NAME: b"child.t"} -``` - -- [ ] **Step 2: Run tests — confirm ImportError** - -```bash -uv run pytest tests/test_hashing/test_schema_cleaner.py -v 2>&1 | head -20 -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.hashing.schema_cleaner'` - -- [ ] **Step 3: Create `src/orcapod/hashing/schema_cleaner.py`** - -```python -"""Schema metadata cleaner for Arrow hashing. - -This module provides utilities for preprocessing Arrow schemas before -content-addressed hashing. Only ``ARROW:extension:*`` metadata keys are -identity-bearing in Orcapod; all other keys (comments, vendor annotations, -source-file provenance, etc.) are stripped so they cannot affect the hash. - -Contract --------- -Only ``ARROW:extension:*`` metadata keys are treated as identity-bearing. -If an extension type were to stamp extra metadata under a non-``ARROW:extension:*`` -key, that metadata would be silently stripped by ``clean_schema_for_hashing``. -Orcapod does not currently rely on any such pattern. - -Public API ----------- -- ``clean_schema_for_hashing(schema)`` — returns a cleaned copy of the schema. -- ``has_extension_metadata(schema)`` — returns True if the (cleaned) schema has - any ``ARROW:extension:name`` key on any field at any nesting depth. -""" - -from __future__ import annotations - -import pyarrow as pa - -_EXTENSION_PREFIX: bytes = b"ARROW:extension:" -_EXTENSION_NAME_KEY: bytes = b"ARROW:extension:name" - - -# --------------------------------------------------------------------------- -# Private helpers -# --------------------------------------------------------------------------- - - -def _filter_metadata(metadata: dict[bytes, bytes] | None) -> dict[bytes, bytes]: - """Return only the entries whose key starts with ``b'ARROW:extension:'``.""" - if not metadata: - return {} - return {k: v for k, v in metadata.items() if k.startswith(_EXTENSION_PREFIX)} - - -def _clean_type(arrow_type: pa.DataType) -> pa.DataType: - """Recursively rebuild a composite type with cleaned child-field metadata. - - For primitive (leaf) types, returns the type unchanged. For composite types - (struct, list, large_list, fixed_size_list, map), rebuilds the type using - cleaned versions of all child fields. - """ - if pa.types.is_struct(arrow_type): - return pa.struct([ - _clean_field(arrow_type.field(i)) - for i in range(arrow_type.num_fields) - ]) - if pa.types.is_list(arrow_type): - return pa.list_(_clean_field(arrow_type.value_field)) - if pa.types.is_large_list(arrow_type): - return pa.large_list(_clean_field(arrow_type.value_field)) - if pa.types.is_fixed_size_list(arrow_type): - return pa.list_(_clean_field(arrow_type.value_field), arrow_type.list_size) - if pa.types.is_map(arrow_type): - return pa.map_( - _clean_field(arrow_type.key_field), - _clean_field(arrow_type.item_field), - arrow_type.keys_sorted, - ) - return arrow_type - - -def _clean_field(field: pa.Field) -> pa.Field: - """Return a copy of ``field`` with metadata and child-type metadata cleaned. - - The field name, type identity (Arrow type), and nullability flag are - preserved exactly. Only the metadata dict is filtered. - """ - return pa.field( - field.name, - _clean_type(field.type), - nullable=field.nullable, - metadata=_filter_metadata(field.metadata), - ) - - -def _has_extension_in_type(arrow_type: pa.DataType) -> bool: - """Return True if any child field of ``arrow_type`` has extension metadata.""" - if pa.types.is_struct(arrow_type): - return any( - _has_extension_in_field(arrow_type.field(i)) - for i in range(arrow_type.num_fields) - ) - if pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type) or pa.types.is_fixed_size_list(arrow_type): - return _has_extension_in_field(arrow_type.value_field) - if pa.types.is_map(arrow_type): - return ( - _has_extension_in_field(arrow_type.key_field) - or _has_extension_in_field(arrow_type.item_field) - ) - return False - - -def _has_extension_in_field(field: pa.Field) -> bool: - """Return True if ``field`` or any of its nested fields carries extension metadata.""" - if field.metadata and _EXTENSION_NAME_KEY in field.metadata: - return True - return _has_extension_in_type(field.type) - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def clean_schema_for_hashing(schema: pa.Schema) -> pa.Schema: - """Return a copy of ``schema`` with all non-extension metadata stripped. - - Walks the schema and every field recursively (through struct, list, - large_list, fixed_size_list, and map child fields). Retains only metadata - keys that start with ``b'ARROW:extension:'`` — at both schema level and - per-field level. All other metadata is dropped. - - Names, types, and nullability flags are untouched. - - Args: - schema: The ``pa.Schema`` to clean. - - Returns: - A new ``pa.Schema`` with the same structure but with non-extension - metadata removed at every level. Schema-level metadata becomes an - empty dict if no extension keys were present. - """ - return pa.schema( - [_clean_field(schema.field(i)) for i in range(len(schema))], - metadata=_filter_metadata(schema.metadata), - ) - - -def has_extension_metadata(schema: pa.Schema) -> bool: - """Return True if ``schema`` has any ``ARROW:extension:name`` key at any depth. - - Intended to be called on the output of ``clean_schema_for_hashing`` so - that the check is purely a key-presence test (no re-filtering needed). - - Recurses through struct, list, large_list, fixed_size_list, and map child - fields identically to ``clean_schema_for_hashing``. - - Args: - schema: A ``pa.Schema`` (typically the output of - ``clean_schema_for_hashing``). - - Returns: - True if any field at any nesting depth carries ``ARROW:extension:name`` - in its metadata; False otherwise. - """ - if schema.metadata and _EXTENSION_NAME_KEY in schema.metadata: - return True - return any( - _has_extension_in_field(schema.field(i)) for i in range(len(schema)) - ) -``` - -- [ ] **Step 4: Run the cleaner tests** - -```bash -uv run pytest tests/test_hashing/test_schema_cleaner.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/hashing/schema_cleaner.py tests/test_hashing/test_schema_cleaner.py -git commit -m "feat(hashing): add clean_schema_for_hashing and has_extension_metadata (PLT-1737)" -``` - ---- - -## Task 3: Add `has_extension_metadata` tests + fixture snapshot tests - -**Files:** -- Modify: `tests/test_hashing/test_schema_cleaner.py` - -The `TestHasExtensionMetadata` class and the deep-recursion fixture were already written in Task 2's test file. If they passed in Task 2, skip this task. If any were left out, add them now. - -- [ ] **Step 1: Verify `TestHasExtensionMetadata` exists and passes** - -```bash -uv run pytest tests/test_hashing/test_schema_cleaner.py::TestHasExtensionMetadata -v -``` - -Expected: all tests pass. (If the class was omitted in Task 2, add it now — see code below.) - -```python -class TestHasExtensionMetadata: - def test_false_for_no_metadata(self): - schema = pa.schema([pa.field("x", pa.int32())]) - assert has_extension_metadata(schema) is False - - def test_false_for_unrelated_metadata_only(self): - schema = pa.schema([ - pa.field("x", pa.int32(), metadata={_COMMENT: b"hi"}), - ]) - assert has_extension_metadata(schema) is False - - def test_true_when_top_level_field_has_extension_name(self): - schema = pa.schema([ - pa.field("x", pa.int32(), metadata={_EXT_NAME: b"my.type"}), - ]) - assert has_extension_metadata(schema) is True - - def test_true_for_deeply_nested_extension_field(self): - """Extension metadata only on a grandchild field still returns True.""" - schema = pa.schema([ - pa.field("outer", pa.struct([ - pa.field("inner", pa.struct([ - pa.field("gc", pa.int32(), metadata={_EXT_NAME: b"gc.t"}), - ])), - ])), - ]) - assert has_extension_metadata(schema) is True - - def test_false_on_cleaned_extension_free_schema(self): - """After cleaning, an extension-free schema returns False.""" - schema = pa.schema([ - pa.field("x", pa.int32(), metadata={_COMMENT: b"drop me"}), - ]) - cleaned = clean_schema_for_hashing(schema) - assert has_extension_metadata(cleaned) is False - - def test_true_for_extension_in_list_value_field(self): - """Extension metadata on a list's value field is detected.""" - schema = pa.schema([ - pa.field("lst", pa.list_( - pa.field("item", pa.int32(), metadata={_EXT_NAME: b"item.t"}), - )), - ]) - assert has_extension_metadata(schema) is True - - def test_true_for_extension_in_map_item_field(self): - schema = pa.schema([ - pa.field("m", pa.map_( - pa.string(), - pa.field("value", pa.int32(), metadata={_EXT_NAME: b"val.t"}), - )), - ]) - assert has_extension_metadata(schema) is True -``` - -- [ ] **Step 2: Run full test suite for schema_cleaner** - -```bash -uv run pytest tests/test_hashing/test_schema_cleaner.py -v -``` - -Expected: all tests pass. - ---- - -## Task 4: Wire `StarfixArrowHasher` (TDD) - -**Files:** -- Modify: `tests/test_hashing/test_starfix_arrow_hasher.py` -- Modify: `src/orcapod/hashing/arrow_hashers.py` - -### Background - -`StarfixArrowHasher.hash_schema` currently calls `ArrowDigester.hash_schema(schema)` (no `include_metadata`). After this task it will: -1. Clean the schema with `clean_schema_for_hashing`. -2. Check `has_extension_metadata` on the cleaned schema. -3. Call `ArrowDigester.hash_schema(clean, include_metadata=include_meta)`. - -`hash_table` similarly cleans the schema of the processed table before calling `ArrowDigester.hash_table`. - -**Stability invariant:** For schemas with no extension metadata, `clean_schema_for_hashing` is a no-op, `has_extension_metadata` returns `False`, and `ArrowDigester.hash_schema(schema, include_metadata=False)` is byte-for-byte identical to the old `ArrowDigester.hash_schema(schema)` call (the v0.3 default is `False`). The existing golden-value tests must still pass. - -- [ ] **Step 1: Write failing hash-path tests** - -Append to `tests/test_hashing/test_starfix_arrow_hasher.py`: - -```python -# --------------------------------------------------------------------------- -# Extension-aware hash path (PLT-1737) -# --------------------------------------------------------------------------- - -_EXT_NAME_KEY = b"ARROW:extension:name" -_COMMENT_KEY = b"comment" - - -class TestHashSchemaExtensionAware: - def test_extension_free_golden_value_unchanged(self): - """Stability invariant: extension-free schema hash is byte-identical to - the pre-v0.3.0 golden value (include_metadata=False is the default).""" - schema = pa.schema( - [ - pa.field("id", pa.int32(), nullable=False), - pa.field("value", pa.float64(), nullable=True), - ] - ) - h = _make_hasher().hash_schema(schema) - assert h.digest.hex() == "000001d676ef0263a8e0e7500b1c97033993dbe445172ca0f9e7577b3994bfa6224b4c", ( - f"Stability invariant broken — extension-free golden digest changed. Got: {h.digest.hex()}" - ) - - def test_mixed_metadata_hash_equals_extension_only_hash(self): - """Key property: a schema with ARROW:extension:name plus unrelated metadata - hashes identically to the same schema with only the extension key.""" - ext_only_schema = pa.schema([ - pa.field("t", pa.binary(), metadata={_EXT_NAME_KEY: b"my.Type"}), - ]) - mixed_schema = pa.schema([ - pa.field("t", pa.binary(), metadata={ - _EXT_NAME_KEY: b"my.Type", - _COMMENT_KEY: b"ignored", - }), - ]) - h_ext = _make_hasher().hash_schema(ext_only_schema) - h_mixed = _make_hasher().hash_schema(mixed_schema) - assert h_ext.digest == h_mixed.digest - - def test_metadata_key_ordering_does_not_affect_hash(self): - """Reordering metadata keys in the input does not change the hash.""" - s1 = pa.schema([pa.field("t", pa.binary(), metadata={ - _EXT_NAME_KEY: b"my.Type", - b"ARROW:extension:metadata": b"{}", - })]) - s2 = pa.schema([pa.field("t", pa.binary(), metadata={ - b"ARROW:extension:metadata": b"{}", - _EXT_NAME_KEY: b"my.Type", - })]) - assert _make_hasher().hash_schema(s1).digest == _make_hasher().hash_schema(s2).digest - - def test_extension_metadata_affects_hash(self): - """Changing an ARROW:extension:name value changes the hash.""" - s1 = pa.schema([pa.field("t", pa.binary(), metadata={_EXT_NAME_KEY: b"TypeA"})]) - s2 = pa.schema([pa.field("t", pa.binary(), metadata={_EXT_NAME_KEY: b"TypeB"})]) - assert _make_hasher().hash_schema(s1).digest != _make_hasher().hash_schema(s2).digest - - def test_unrelated_metadata_does_not_affect_hash(self): - """Adding only unrelated metadata keys does not change the hash.""" - s_clean = pa.schema([pa.field("x", pa.int32())]) - s_noise = pa.schema([pa.field("x", pa.int32(), metadata={_COMMENT_KEY: b"hi"})]) - assert _make_hasher().hash_schema(s_clean).digest == _make_hasher().hash_schema(s_noise).digest - - -class TestHashTableExtensionAware: - def test_extension_free_table_golden_value_unchanged(self): - """Stability invariant: extension-free table hash is byte-identical to - the pre-v0.3.0 golden value.""" - table = pa.table({ - "id": pa.array([1, 2, 3], type=pa.int32()), - "score": pa.array([0.1, 0.2, 0.3], type=pa.float64()), - "label": pa.array(["a", "b", "c"], type=pa.utf8()), - }) - h = _make_hasher().hash_table(table) - assert h.digest.hex() == "0000010cd7fe5462420b84f03a06925374e528817a3b72319e679a17e7380964878791", ( - f"Stability invariant broken — extension-free table golden digest changed. Got: {h.digest.hex()}" - ) - - def test_table_mixed_schema_metadata_equals_extension_only(self): - """Table schema with mixed metadata hashes identically to same table - schema with only the extension key (unrelated keys are stripped).""" - schema_ext_only = pa.schema([ - pa.field("x", pa.int32(), metadata={_EXT_NAME_KEY: b"my.Type"}), - ]) - schema_mixed = pa.schema([ - pa.field("x", pa.int32(), metadata={ - _EXT_NAME_KEY: b"my.Type", - _COMMENT_KEY: b"ignored", - }), - ]) - data = [pa.array([1, 2, 3], type=pa.int32())] - t_ext = pa.table(data, schema=schema_ext_only) - t_mixed = pa.table(data, schema=schema_mixed) - assert _make_hasher().hash_table(t_ext).digest == _make_hasher().hash_table(t_mixed).digest - - def test_table_unrelated_schema_metadata_does_not_affect_hash(self): - """Adding only unrelated schema metadata does not change the table hash.""" - t_clean = pa.table({"x": pa.array([1, 2], type=pa.int32())}) - schema_noise = pa.schema([pa.field("x", pa.int32(), metadata={_COMMENT_KEY: b"hi"})]) - t_noise = pa.table([pa.array([1, 2], type=pa.int32())], schema=schema_noise) - assert _make_hasher().hash_table(t_clean).digest == _make_hasher().hash_table(t_noise).digest -``` - -- [ ] **Step 2: Run new tests — confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_starfix_arrow_hasher.py::TestHashSchemaExtensionAware tests/test_hashing/test_starfix_arrow_hasher.py::TestHashTableExtensionAware -v 2>&1 | tail -20 -``` - -Expected: several tests FAIL (the `test_mixed_metadata_hash_equals_extension_only_hash` and `test_table_mixed_schema_metadata_equals_extension_only` tests will fail because the cleaner is not yet wired in). - -- [ ] **Step 3: Update `StarfixArrowHasher` in `arrow_hashers.py`** - -At the top of `src/orcapod/hashing/arrow_hashers.py`, add the import after the existing imports: - -```python -from orcapod.hashing.schema_cleaner import clean_schema_for_hashing, has_extension_metadata -``` - -Replace `hash_schema` (lines ~336–351 in the current file): - -```python - def hash_schema(self, schema: pa.Schema) -> ContentHash: - """Hash an Arrow schema using the starfix canonical algorithm. - - The schema is preprocessed by ``clean_schema_for_hashing`` before - hashing: non-``ARROW:extension:*`` metadata is stripped at every - level. ``include_metadata=True`` is passed to ``ArrowDigester`` only - when extension metadata is present after cleaning, preserving - byte-for-byte hash stability with pre-v0.3.0 output for schemas that - carry no extension metadata. - - Parameters - ---------- - schema: - The ``pa.Schema`` to hash. - - Returns - ------- - ContentHash - A ``ContentHash`` whose ``digest`` is the 35-byte versioned - SHA-256 produced by ``ArrowDigester.hash_schema``. - """ - clean = clean_schema_for_hashing(schema) - include_meta = has_extension_metadata(clean) - digest = ArrowDigester.hash_schema(clean, include_metadata=include_meta) - return ContentHash(method=self._hasher_id, digest=digest) -``` - -Replace `hash_table` (lines ~353–377 in the current file): - -```python - def hash_table(self, table: pa.Table | pa.RecordBatch) -> ContentHash: - """Hash an Arrow table (or ``RecordBatch``) using starfix. - - Semantic types are resolved to their content-hash strings before the - table is passed to ``ArrowDigester.hash_table``. The table's schema - is then preprocessed by ``clean_schema_for_hashing`` to strip - non-``ARROW:extension:*`` metadata. ``include_metadata=True`` is - passed to ``ArrowDigester`` only when extension metadata is present - after cleaning. - - Parameters - ---------- - table: - The ``pa.Table`` or ``pa.RecordBatch`` to hash. - - Returns - ------- - ContentHash - A ``ContentHash`` whose ``digest`` is the 35-byte versioned - SHA-256 produced by ``ArrowDigester.hash_table``. - """ - if isinstance(table, pa.RecordBatch): - table = pa.Table.from_batches([table]) - - processed_table = self._process_table_columns(table) - clean_schema = clean_schema_for_hashing(processed_table.schema) - clean_table = pa.Table.from_arrays( - processed_table.columns, schema=clean_schema - ) - include_meta = has_extension_metadata(clean_schema) - digest = ArrowDigester.hash_table(clean_table, include_metadata=include_meta) - return ContentHash(method=self._hasher_id, digest=digest) -``` - -- [ ] **Step 4: Run the full hashing test suite** - -```bash -uv run pytest tests/test_hashing/ -v -``` - -Expected: all tests pass, including the pre-existing golden-value tests in `TestHashSchema` and `TestHashTable`. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/hashing/arrow_hashers.py tests/test_hashing/test_starfix_arrow_hasher.py -git commit -m "feat(hashing): wire schema cleaner into StarfixArrowHasher (PLT-1737)" -``` - ---- - -## Task 5: Run the full test suite - -- [ ] **Step 1: Run all tests** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -30 -``` - -Expected: all tests pass. If any pre-existing tests fail, investigate before proceeding — do not skip. - -- [ ] **Step 2: Commit if any fixes were needed** - -If no fixes were needed, no additional commit is required. - ---- - -## Task 6: Update CHANGELOG.md - -**Files:** -- Modify: `CHANGELOG.md` - -- [ ] **Step 1: Add entry under `## [Unreleased]`** - -Insert the following block immediately after the `## [Unreleased]` heading (before any existing entries): - -```markdown -### Changed - -#### starfix v0.3.0 adoption + schema-metadata cleaner (PLT-1737) - -Bumped `starfix` dependency to `~=0.3.0` and introduced a schema-metadata -cleaning step before all Arrow schema and table hashes. - -**What changed:** `StarfixArrowHasher.hash_schema` and `hash_table` now strip -every metadata key that does not start with `ARROW:extension:` before passing -the schema to starfix. Only identity-bearing extension metadata (e.g. -`ARROW:extension:name`) is included in the hash. Unrelated keys (comments, -vendor annotations, source-file provenance, etc.) are ignored. - -**Stability invariant:** Schemas with no `ARROW:extension:*` metadata anywhere -continue to produce byte-for-byte identical hashes to pre-v0.3.0 Orcapod. - -**One-time hash invalidation:** Pipelines whose schemas contained -`ARROW:extension:*` keys *alongside* unrelated field metadata (e.g. a -`comment` key on the same field as `ARROW:extension:name`) will see a changed -hash. On-disk caches for those pipelines should be treated as stale and -recomputed. - -**New utility:** `orcapod.hashing.schema_cleaner.clean_schema_for_hashing` and -`has_extension_metadata` are available as semi-public utilities for other -Orcapod code that needs to inspect or clean Arrow schema metadata. -``` - -- [ ] **Step 2: Commit** - -```bash -git add CHANGELOG.md -git commit -m "docs(changelog): add starfix v0.3.0 adoption and schema cleaner note (PLT-1737)" -``` - ---- - -## Task 7: Final verification - -- [ ] **Step 1: Run the full test suite one last time** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -10 -``` - -Expected output ends with: -``` -========== N passed in X.XXs ========== -``` - -No failures, no errors. - -- [ ] **Step 2: Confirm branch and diff** - -```bash -git log --oneline main..HEAD -git diff main --stat -``` - -Expected log (4 commits): -``` - docs(changelog): add starfix v0.3.0 adoption and schema cleaner note (PLT-1737) - feat(hashing): wire schema cleaner into StarfixArrowHasher (PLT-1737) - feat(hashing): add clean_schema_for_hashing and has_extension_metadata (PLT-1737) - chore(deps): pin starfix to ~=0.3.0 (PLT-1737) -``` - -Expected diff stat touches only: -- `pyproject.toml` -- `uv.lock` -- `src/orcapod/hashing/schema_cleaner.py` (new) -- `src/orcapod/hashing/arrow_hashers.py` -- `tests/test_hashing/test_schema_cleaner.py` (new) -- `tests/test_hashing/test_starfix_arrow_hasher.py` -- `CHANGELOG.md` diff --git a/superpowers/plans/2026-06-23-plt-1659-extension-type-roundtrip-integration-tests.md b/superpowers/plans/2026-06-23-plt-1659-extension-type-roundtrip-integration-tests.md deleted file mode 100644 index 0028fb2c5..000000000 --- a/superpowers/plans/2026-06-23-plt-1659-extension-type-roundtrip-integration-tests.md +++ /dev/null @@ -1,827 +0,0 @@ -# PLT-1659: Extension Type Round-Trip Integration Tests — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add three new integration test files covering end-to-end extension type round-trips through Parquet, Delta Lake, schema compatibility, and per-process cache behaviour. - -**Architecture:** Three focused test files plus one source change and one docs update. Test files: `test_roundtrips.py` (write/read through Parquet and Delta backends), `test_schema_compatibility.py` (Arrow-level identity + Python-type-level compatibility), `test_cache_behavior.py` (registry cache populated and skipped on second read). SQLite backend is excluded from value round-trip tests because `SQLiteConnector` does not preserve `ARROW:extension:*` field metadata; that pattern is already covered by `test_extension_aware_database.py`. Source change: `ConnectorArrowDatabase.add_records()` gets a `ValueError` guard that rejects extension-typed columns (both in-memory `pa.ExtensionType` and metadata-only fields) as an interim safety measure while PLT-1795 is pending. - -**Tech Stack:** pytest, pyarrow, pyarrow.parquet, deltalake, polars, orcapod extension type APIs (`create_registry`, `UniversalTypeConverter`, `DataclassLogicalTypeFactory`), `unittest.mock.patch.object`. - ---- - -## File Map - -| Action | Path | -|---|---| -| Create | `tests/test_extension_types/test_schema_compatibility.py` | -| Create | `tests/test_extension_types/test_cache_behavior.py` | -| Create | `tests/test_extension_types/test_roundtrips.py` | -| Modify | `src/orcapod/databases/connector_arrow_database.py` — add `ValueError` guard in `add_records()` | -| Modify | `DESIGN_ISSUES.md` — add CA1 entry documenting SQL metadata loss and interim guard | - ---- - -## Task 1: Create and check out the feature branch - -**Files:** none (git only) - -- [ ] **Step 1: Verify you are on `extension-type-system`** - -```bash -git branch --show-current -``` - -Expected output: `extension-type-system` - -- [ ] **Step 2: Create and check out the feature branch** - -```bash -git checkout -b eywalker/plt-1659-integration-tests-end-to-end-semantic-type-round-trips -git branch --show-current -``` - -Expected output: `eywalker/plt-1659-integration-tests-end-to-end-semantic-type-round-trips` - ---- - -## Task 2: `test_schema_compatibility.py` - -**Files:** -- Create: `tests/test_extension_types/test_schema_compatibility.py` - -This file has no backend dependencies — it only needs a fresh `UniversalTypeConverter` and `check_schema_compatibility`. - -- [ ] **Step 1: Write the test file** - -Create `tests/test_extension_types/test_schema_compatibility.py` with this exact content: - -```python -"""Integration tests for extension-type-backed schema compatibility. - -Two complementary angles: - -Arrow-level identity - ``converter.python_schema_to_arrow_schema`` assigns each dataclass a unique - Arrow extension name derived from its fully-qualified class name. Two - dataclasses with identical struct shapes but different class names therefore - produce *different* extension names — the core identity guarantee of the - extension type system. - -Python-type-level compatibility - ``check_schema_compatibility`` from ``schema_utils`` uses beartype - ``is_subhint`` to compare Python type annotations. Same class → compatible; - different class with the same struct shape → incompatible. This is the - property that prevents silent data corruption when two unrelated dataclasses - happen to share the same fields. -""" -from __future__ import annotations - -import dataclasses - -import pyarrow as pa - -from orcapod.contexts import create_registry -from orcapod.types import Schema -from orcapod.utils.schema_utils import check_schema_compatibility - - -# Module-level dataclasses — DataclassLogicalTypeFactory rejects local classes -# because they have no stable fully-qualified class name for reconstruction. - -@dataclasses.dataclass -class _PointA: - x: int - y: int - - -@dataclasses.dataclass -class _PointB: - """Same struct shape as _PointA but a different class name.""" - x: int - y: int - - -# ── Arrow-level identity tests ──────────────────────────────────────────────── - - -def test_arrow_schema_distinct_extension_names_for_same_shape(): - """_PointA and _PointB produce different Arrow extension names despite identical shapes. - - This is the core identity guarantee: struct shape alone does not determine - type identity in the extension type system. - """ - converter_a = create_registry().get_context().type_converter - converter_b = create_registry().get_context().type_converter - - type_a = converter_a.register_python_class(_PointA) - type_b = converter_b.register_python_class(_PointB) - - assert isinstance(type_a, pa.ExtensionType) - assert isinstance(type_b, pa.ExtensionType) - - fqcn_a = f"{_PointA.__module__}.{_PointA.__qualname__}" - fqcn_b = f"{_PointB.__module__}.{_PointB.__qualname__}" - assert type_a.extension_name == fqcn_a - assert type_b.extension_name == fqcn_b - assert type_a.extension_name != type_b.extension_name - - -def test_arrow_schema_same_extension_name_idempotent(): - """Registering _PointA twice returns the same extension name both times.""" - converter = create_registry().get_context().type_converter - - type_first = converter.register_python_class(_PointA) - type_second = converter.register_python_class(_PointA) - - assert isinstance(type_first, pa.ExtensionType) - assert isinstance(type_second, pa.ExtensionType) - assert type_first.extension_name == type_second.extension_name - - -# ── Python-type-level compatibility tests ───────────────────────────────────── - - -def test_python_schema_compatibility_passes_same_type(): - """Incoming _PointA is compatible with receiving _PointA.""" - result = check_schema_compatibility( - {"value": _PointA}, - Schema({"value": _PointA}), - ) - assert result is True - - -def test_python_schema_compatibility_rejects_different_type_same_shape(): - """Incoming _PointA is NOT compatible with receiving _PointB. - - Both dataclasses share the same struct shape {x: int, y: int}, but they - are different Python types. The old shape-based system would have accepted - this silently; the extension type system correctly rejects it. - """ - result = check_schema_compatibility( - {"value": _PointA}, - Schema({"value": _PointB}), - ) - assert result is False -``` - -- [ ] **Step 2: Run the tests and verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_schema_compatibility.py -v -``` - -Expected: all 4 tests pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_schema_compatibility.py -git commit -m "test(extension-types): add schema compatibility integration tests (PLT-1659)" -``` - ---- - -## Task 3: `test_cache_behavior.py` - -**Files:** -- Create: `tests/test_extension_types/test_cache_behavior.py` - -Uses Parquet as the storage backend (simplest — no database wrapper needed). The second test patches `DataclassLogicalTypeFactory.reconstruct_from_arrow` at the class level to count calls; `wraps=` preserves the original behaviour so the test still exercises the real code path. - -- [ ] **Step 1: Write the test file** - -Create `tests/test_extension_types/test_cache_behavior.py` with this exact content: - -```python -"""Integration tests for per-process extension type cache behaviour. - -The ``LogicalTypeRegistry`` stores registered types in an in-memory dict keyed -by Arrow extension name. ``register_discovered_extensions`` skips the factory -call (``reconstruct_from_arrow``) when the extension name is already present in -the registry — this is the "cache hit" path. - -Two tests: - -1. ``test_cache_populated_after_first_read`` — verifies the type is absent from - a fresh converter's registry before reading a Parquet file, and present after. - -2. ``test_factory_not_called_on_second_read`` — verifies that ``reconstruct_from_arrow`` - is called exactly once (first read) and zero additional times on the second - read of the same file. -""" -from __future__ import annotations - -import dataclasses -from unittest.mock import patch - -import pyarrow as pa -import pyarrow.parquet as pq - -from orcapod.contexts import create_registry -from orcapod.extension_types.dataclass_logical_type_factory import DataclassLogicalTypeFactory - - -# Module-level dataclass — local classes cannot be reconstructed from FQCN. - -@dataclasses.dataclass -class _CachePoint: - x: int - y: int - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - - -def _fresh_converter(): - """Return a fresh UniversalTypeConverter from a new registry instance. - - Uses ``create_registry()`` instead of ``get_default_context()`` to avoid - cross-test contamination through the global singleton cache. - """ - return create_registry().get_context().type_converter - - -def _write_parquet(tmp_path, converter) -> str: - """Write a _CachePoint column to Parquet and return the file path as str.""" - converter.register_python_class(_CachePoint) - arrow_schema = converter.python_schema_to_arrow_schema({"point": _CachePoint}) - rows = [{"point": _CachePoint(x=1, y=2)}] - table = converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - parquet_path = tmp_path / "cache_test.parquet" - pq.write_table(table, str(parquet_path)) - return str(parquet_path) - - -# ── Tests ───────────────────────────────────────────────────────────────────── - - -def test_cache_populated_after_first_read(tmp_path): - """Registry has _CachePoint after load_extension_types on a fresh converter. - - Before reading: the fresh converter's registry does not know about _CachePoint. - After reading: register_discovered_extensions triggers reconstruct_from_arrow - which registers _CachePoint, populating the cache. - """ - write_converter = _fresh_converter() - parquet_path = _write_parquet(tmp_path, write_converter) - - read_converter = _fresh_converter() - fqcn = f"{_CachePoint.__module__}.{_CachePoint.__qualname__}" - - # Before read: not registered - assert read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn) is None - - read_converter.load_extension_types(pq.read_table(parquet_path)) - - # After read: registered (cache populated) - assert read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn) is not None - - -def test_factory_not_called_on_second_read(tmp_path): - """reconstruct_from_arrow called once on first read, zero times on second read. - - On first read, register_discovered_extensions finds _CachePoint's extension - name in the schema, dispatches to the factory (call count = 1), and stores - the result in the registry. - - On second read, register_discovered_extensions finds the extension name already - in the registry and short-circuits — the factory is not called again - (call count remains 1). - """ - write_converter = _fresh_converter() - parquet_path = _write_parquet(tmp_path, write_converter) - - read_converter = _fresh_converter() - - with patch.object( - DataclassLogicalTypeFactory, - "reconstruct_from_arrow", - autospec=True, - wraps=DataclassLogicalTypeFactory.reconstruct_from_arrow, - ) as spy: - # First read: factory is called once - read_converter.load_extension_types(pq.read_table(parquet_path)) - assert spy.call_count == 1, f"Expected 1 factory call, got {spy.call_count}" - - # Second read on the same file: registry hit — factory not called again - read_converter.load_extension_types(pq.read_table(parquet_path)) - assert spy.call_count == 1, ( - f"Expected still 1 factory call after second read, got {spy.call_count}" - ) -``` - -- [ ] **Step 2: Run the tests and verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_cache_behavior.py -v -``` - -Expected: both tests pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_cache_behavior.py -git commit -m "test(extension-types): add per-process cache behaviour integration tests (PLT-1659)" -``` - ---- - -## Task 4: `test_roundtrips.py` — backend fixture + all parametrised tests - -**Files:** -- Create: `tests/test_extension_types/test_roundtrips.py` - -**Important note on SQLite:** `SQLiteConnector` maps Arrow types to SQL column types and does not preserve `ARROW:extension:*` field metadata. `ExtensionAwareDatabase` relies on that metadata to auto-register and re-wrap extension types on read. Without it, `apply_extension_types` is a no-op and values are returned as plain storage scalars (string, bytes, dict). SQLite backend round-trip tests are therefore omitted from this file; the `ExtensionAwareDatabase` wrapper behaviour is already covered by `tests/test_databases/test_extension_aware_database.py`. - -The Parquet and Delta backends both preserve field metadata (through the Arrow → Parquet encoding) and fully support the peek-register-read pattern. - -- [ ] **Step 1: Write the test file** - -Create `tests/test_extension_types/test_roundtrips.py` with this exact content: - -```python -"""End-to-end integration tests for extension type round-trips. - -Tests the complete pipeline: - - Python object → write → storage → peek-schema → register → read → Python object - -Each round-trip test is parameterised over two storage backends: - -- ``parquet``: direct ``pyarrow.parquet`` write/read. -- ``delta``: ``deltalake.write_deltalake`` / ``DeltaTable.to_pyarrow_dataset(as_large_types=True).to_table()``. - -SQLite (``ConnectorArrowDatabase`` + ``SQLiteConnector``) is excluded because -``SQLiteConnector`` maps Arrow types to SQL column types and discards -``ARROW:extension:*`` field metadata. Without that metadata, the -peek-register-read pattern cannot auto-register extension types on the read -path. The ``ExtensionAwareDatabase`` wrapper behaviour over SQLite is already -tested in ``tests/test_databases/test_extension_aware_database.py``. -""" -from __future__ import annotations - -import dataclasses -import pathlib -import uuid as uuid_module -from pathlib import Path -from typing import Callable - -import pyarrow as pa -import pyarrow.parquet as pq -import pytest -from upath import UPath - -from orcapod.contexts import create_registry -from orcapod.semantic_types.universal_converter import UniversalTypeConverter - - -# ── Module-level dataclasses ────────────────────────────────────────────────── -# DataclassLogicalTypeFactory rejects local (in-function) classes because they -# have no stable fully-qualified class name for reconstruction from Arrow schema. - -@dataclasses.dataclass -class _PointA: - x: int - y: int - - -@dataclasses.dataclass -class _PointB: - """Same struct shape as _PointA, different class name.""" - x: int - y: int - - -@dataclasses.dataclass -class _Inner: - value: int - - -@dataclasses.dataclass -class _Outer: - inner: _Inner - label: str - - -# ── Storage backend abstraction ─────────────────────────────────────────────── - - -@dataclasses.dataclass -class _StorageBackend: - """Encapsulates backend-specific write and read logic for parameterised tests. - - Args: - name: Short identifier used in pytest test IDs (e.g. ``"parquet"``). - write: Callable that writes an Arrow table to a directory. - read: Callable that reads from that directory and returns an Arrow table - with extension types registered and applied. Must return only the - original user data columns (no ``__record_id`` or similar). - """ - name: str - write: Callable[[pa.Table, Path], None] - read: Callable[[Path, UniversalTypeConverter], pa.Table] - - -def _parquet_write(table: pa.Table, base_path: Path) -> None: - pq.write_table(table, str(base_path / "data.parquet")) - - -def _parquet_read(base_path: Path, converter: UniversalTypeConverter) -> pa.Table: - return converter.load_extension_types(pq.read_table(str(base_path / "data.parquet"))) - - -def _delta_write(table: pa.Table, base_path: Path) -> None: - import deltalake - deltalake.write_deltalake(str(base_path / "delta"), table) - - -def _delta_read(base_path: Path, converter: UniversalTypeConverter) -> pa.Table: - import deltalake - dt = deltalake.DeltaTable(str(base_path / "delta")) - # as_large_types=True preserves large_string / large_binary rather than - # normalising them to string / binary (Delta Lake's default behaviour). - raw = dt.to_pyarrow_dataset(as_large_types=True).to_table() - return converter.load_extension_types(raw) - - -_BACKENDS = [ - _StorageBackend(name="parquet", write=_parquet_write, read=_parquet_read), - _StorageBackend(name="delta", write=_delta_write, read=_delta_read), -] - - -@pytest.fixture(params=_BACKENDS, ids=lambda b: b.name) -def storage_backend(request: pytest.FixtureRequest) -> _StorageBackend: - """Yield one storage backend per parametrised run.""" - return request.param - - -# ── Internal helpers ────────────────────────────────────────────────────────── - - -def _fresh_converter() -> UniversalTypeConverter: - """Return a fresh converter from a new registry instance. - - Uses ``create_registry()`` instead of ``get_default_context()`` to avoid - cross-test contamination through the global singleton cache. - """ - return create_registry().get_context().type_converter - - -def _write_and_read( - schema_dict: dict, - rows: list[dict], - backend: _StorageBackend, - tmp_path: Path, -) -> tuple[pa.Table, UniversalTypeConverter]: - """Write rows with a fresh write converter and read back with a fresh read converter. - - Returns the resulting Arrow table (with extension types applied) and the - read-side converter (needed for ``arrow_table_to_python_dicts``). - """ - write_converter = _fresh_converter() - arrow_schema = write_converter.python_schema_to_arrow_schema(schema_dict) - table = write_converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - backend.write(table, tmp_path) - - read_converter = _fresh_converter() - result = backend.read(tmp_path, read_converter) - return result, read_converter - - -# ── Built-in type round-trip tests ─────────────────────────────────────────── - - -def test_builtin_path_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """pathlib.Path round-trips through storage with extension name ``orcapod.path``. - - Built-in types (Path, UPath, UUID) are pre-registered in the default context - so the read-side converter already knows about them. The test verifies that: - - 1. The Arrow field carries the ``orcapod.path`` extension type after read. - 2. The Python value is reconstructed as a ``pathlib.Path`` instance. - """ - p = pathlib.Path("/tmp/orcapod/integration/test.txt") - result, read_converter = _write_and_read( - {"col": pathlib.Path}, - [{"col": p}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("col") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'col', got plain type {field.type!r}" - ) - assert field.type.extension_name == "orcapod.path" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert isinstance(rows[0]["col"], pathlib.Path) - assert rows[0]["col"] == p - - -def test_builtin_upath_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """UPath round-trips through storage with extension name ``orcapod.upath``.""" - u = UPath("s3://my-bucket/data/file.parquet") - result, read_converter = _write_and_read( - {"col": UPath}, - [{"col": u}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("col") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'col', got plain type {field.type!r}" - ) - assert field.type.extension_name == "orcapod.upath" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert isinstance(rows[0]["col"], UPath) - assert str(rows[0]["col"]) == str(u) - - -def test_builtin_uuid_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """uuid.UUID round-trips through storage with extension name ``orcapod.uuid``.""" - u = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - result, read_converter = _write_and_read( - {"col": uuid_module.UUID}, - [{"col": u}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("col") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'col', got plain type {field.type!r}" - ) - assert field.type.extension_name == "orcapod.uuid" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert isinstance(rows[0]["col"], uuid_module.UUID) - assert rows[0]["col"] == u - - -# ── Dataclass round-trip tests ──────────────────────────────────────────────── - - -def test_simple_dataclass_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """Simple dataclass round-trips with correct FQCN as the Arrow extension name. - - The read-side converter starts with no knowledge of _PointA. After read, - register_discovered_extensions triggers DataclassLogicalTypeFactory which - imports _PointA from its fully-qualified class name and registers it. - """ - point = _PointA(x=3, y=7) - result, read_converter = _write_and_read( - {"point": _PointA}, - [{"point": point}], - storage_backend, - tmp_path, - ) - - fqcn = f"{_PointA.__module__}.{_PointA.__qualname__}" - field = result.schema.field("point") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'point', got {field.type!r}" - ) - assert field.type.extension_name == fqcn - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - reconstructed = rows[0]["point"] - assert isinstance(reconstructed, _PointA) - assert reconstructed.x == 3 - assert reconstructed.y == 7 - - -def test_two_dataclasses_same_shape_distinct_extension_names( - storage_backend: _StorageBackend, tmp_path: Path -) -> None: - """_PointA and _PointB have the same struct shape but different extension names. - - Writing _PointA and reading it back must NOT reconstruct a _PointB, even - though their on-disk struct shapes (x: int, y: int) are identical. The - extension name (FQCN) is the sole identity signal. - """ - point_a = _PointA(x=1, y=2) - result, read_converter = _write_and_read( - {"point": _PointA}, - [{"point": point_a}], - storage_backend, - tmp_path, - ) - - fqcn_a = f"{_PointA.__module__}.{_PointA.__qualname__}" - fqcn_b = f"{_PointB.__module__}.{_PointB.__qualname__}" - - field = result.schema.field("point") - assert hasattr(field.type, "extension_name") - assert field.type.extension_name == fqcn_a - assert field.type.extension_name != fqcn_b # distinct from _PointB - - rows = read_converter.arrow_table_to_python_dicts(result) - reconstructed = rows[0]["point"] - assert isinstance(reconstructed, _PointA) - assert not isinstance(reconstructed, _PointB) - - -def test_nested_dataclass_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """Nested dataclass: _Outer and _Inner both registered; full object reconstructed. - - register_discovered_extensions triggers DataclassLogicalTypeFactory for _Outer. - That factory's reconstruct_from_arrow calls converter.register_python_class(_Inner) - as a side-effect, so _Inner is also registered without an explicit peek step. - """ - outer = _Outer(inner=_Inner(value=42), label="hello") - result, read_converter = _write_and_read( - {"item": _Outer}, - [{"item": outer}], - storage_backend, - tmp_path, - ) - - fqcn_outer = f"{_Outer.__module__}.{_Outer.__qualname__}" - fqcn_inner = f"{_Inner.__module__}.{_Inner.__qualname__}" - - assert read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn_outer) is not None, ( - "_Outer should be registered after read" - ) - assert read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn_inner) is not None, ( - "_Inner should be registered transitively after read" - ) - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - reconstructed = rows[0]["item"] - assert isinstance(reconstructed, _Outer) - assert isinstance(reconstructed.inner, _Inner) - assert reconstructed.inner.value == 42 - assert reconstructed.label == "hello" -``` - -- [ ] **Step 2: Run the tests and verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py -v -``` - -Expected: all 12 parametrised tests pass (6 test functions × 2 backends). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_roundtrips.py -git commit -m "test(extension-types): add Parquet/Delta round-trip integration tests (PLT-1659)" -``` - ---- - -## Task 5: Add the Delta Polars native-read test to `test_roundtrips.py` - -**Files:** -- Modify: `tests/test_extension_types/test_roundtrips.py` (append one function) - -This test reads a Delta table back via `pl.read_delta` (Polars' native Delta reader) rather than `DeltaTable.to_pyarrow_table()`, verifying that extension type metadata survives the Polars path. - -When the write-side converter calls `register_python_class(_PointA)`, it registers `_PointA` in both PyArrow's and Polars' **global** registries (as a side-effect of `registry.register_logical_type`). That global registration persists for the duration of the test process, so `pl.read_delta` can resolve `_PointA`'s extension type when reading the underlying Parquet files. - -- [ ] **Step 1: Append the Delta Polars test to `test_roundtrips.py`** - -Append the following block at the end of `tests/test_extension_types/test_roundtrips.py`: - -```python -# ── Delta Lake: Polars native read ─────────────────────────────────────────── - - -def test_delta_polars_read_delta(tmp_path: Path) -> None: - """Write a dataclass column to Delta; read back via pl.read_delta; extension type survives. - - The write-side converter registers _PointA in both PyArrow's and Polars' - global registries (``register_python_class`` calls ``make_polars_extension_type`` - which registers with Polars). ``pl.read_delta`` can therefore decode the column - as the correct Polars extension type, not a plain ``Struct``. - - Note: ``pl.DataFrame.to_arrow()`` exports Polars extension types as PyArrow - extension arrays but with empty serialized bytes (Polars does not forward - ``__arrow_ext_metadata__`` through its Arrow export). Python-object - reconstruction via the Polars-to-Arrow path is therefore not possible; that - path is tested by the separate ``parquet`` / ``delta`` parametrised tests - which read underlying Parquet files directly. - """ - import deltalake - import polars as pl - - delta_path = str(tmp_path / "polars_delta") - fqcn = f"{_PointA.__module__}.{_PointA.__qualname__}" - - # Write — registers _PointA in PyArrow + Polars global registries. - write_converter = _fresh_converter() - write_converter.register_python_class(_PointA) - arrow_schema = write_converter.python_schema_to_arrow_schema({"point": _PointA}) - rows = [{"point": _PointA(x=5, y=9)}] - table = write_converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - deltalake.write_deltalake(delta_path, table) - - # Read via Polars native Delta reader. - # _PointA is already in the Polars global registry from the write step above. - df = pl.read_delta(delta_path) - - # Assert the column carries the correct Polars extension type — not a plain Struct. - col_dtype = df.dtypes[0] - assert col_dtype.is_extension(), ( - f"Expected a Polars extension type on column 'point', got {col_dtype!r}" - ) - assert col_dtype.ext_name() == fqcn, ( - f"Expected extension name {fqcn!r}, got {col_dtype.ext_name()!r}" - ) -``` - -- [ ] **Step 2: Run the new test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py::test_delta_polars_read_delta -v -``` - -Expected: 1 test passes. - -- [ ] **Step 3: Run the full roundtrips file to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py -v -``` - -Expected: 13 tests pass (12 from Task 4 + 1 new). - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_extension_types/test_roundtrips.py -git commit -m "test(extension-types): add Delta Polars native-read round-trip test (PLT-1659)" -``` - ---- - -## Task 6: Full test run and PR - -**Files:** none - -- [ ] **Step 1: Run the full extension-types test suite** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all tests pass. The three new files contribute 17 tests: -- `test_schema_compatibility.py`: 4 tests -- `test_cache_behavior.py`: 2 tests -- `test_roundtrips.py`: 13 tests - -- [ ] **Step 2: Run the broader test suite to check for regressions** - -```bash -uv run pytest tests/ -x -q --ignore=tests/test_semantic_types -``` - -Expected: no new failures. (`test_semantic_types/` tests the old shape-based system and is excluded per the PLT-1659 spec.) - -- [ ] **Step 3: Push the branch** - -```bash -git push -u origin eywalker/plt-1659-integration-tests-end-to-end-semantic-type-round-trips -``` - -- [ ] **Step 4: Open the PR** - -```bash -gh pr create \ - --base extension-type-system \ - --title "test(extension-types): end-to-end round-trip integration tests (PLT-1659)" \ - --body "$(cat <<'EOF' -## Summary - -Adds three integration test files covering the full extension type round-trip pipeline: - -- **`test_roundtrips.py`** — write/read round-trips for built-in types (Path, UPath, UUID), simple dataclass, two same-shaped dataclasses with distinct extension names, nested dataclass, and Polars native Delta read. Parameterised over Parquet and Delta backends. -- **`test_schema_compatibility.py`** — Arrow-level extension name identity checks and Python-type-level `check_schema_compatibility` pass/reject tests. -- **`test_cache_behavior.py`** — verifies the per-process registry cache is populated on first read and that `reconstruct_from_arrow` is not called on subsequent reads of the same file. - -## Deferred (noted in corresponding issues) - -- `list[MyDataclass]` round-trip → PLT-1732 (requires `ListLogicalType`) -- Picklable type tests → PLT-1658 (handler not yet implemented) -- SQLite value round-trips → excluded because `SQLiteConnector` does not preserve `ARROW:extension:*` field metadata; `ExtensionAwareDatabase` wrapper already tested in `test_extension_aware_database.py` - -Closes PLT-1659 -EOF -)" -``` - -- [ ] **Step 5: Confirm the PR URL is printed and note it** - -The `gh pr create` command prints the PR URL. Record it for tracking. diff --git a/superpowers/plans/2026-06-23-pyspiral-upgrade-plt-1773.md b/superpowers/plans/2026-06-23-pyspiral-upgrade-plt-1773.md deleted file mode 100644 index b23adc2ed..000000000 --- a/superpowers/plans/2026-06-23-pyspiral-upgrade-plt-1773.md +++ /dev/null @@ -1,316 +0,0 @@ -# pyspiral Upgrade (PLT-1773) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Upgrade `pyspiral` from the broken `0.11.7` to `0.14.9` so that all three -`TestSpiralDBConnectorIntegration` tests pass green in CI again. - -**Architecture:** Three targeted edits — bump the minimum version constraint in -`pyproject.toml`, verify the `uv.lock` pin is at `0.14.9` (already updated during -investigation), and add breadcrumb documentation so the next person to hit a -pyspiral-related failure has context. No connector code changes needed: the API is -fully backward-compatible across this version range. - -**Tech Stack:** Python, uv, pyspiral 0.14.9, pytest - ---- - -### Task 1: Bump the `pyspiral` minimum in `pyproject.toml` - -**Files:** -- Modify: `pyproject.toml:47-49` - -- [ ] **Step 1: Update the constraint** - -In `pyproject.toml`, change line 48 from: - -```toml -[project.optional-dependencies] -redis = ["redis>=6.2.0"] -ray = ["ray[default]==2.48.0", "ipywidgets>=8.1.7"] -postgresql = ["psycopg[binary]>=3.0"] -spiraldb = [ - "pyspiral>=0.11.0", -] -``` - -to: - -```toml -[project.optional-dependencies] -redis = ["redis>=6.2.0"] -ray = ["ray[default]==2.48.0", "ipywidgets>=8.1.7"] -postgresql = ["psycopg[binary]>=3.0"] -spiraldb = [ - "pyspiral>=0.14.0", -] -``` - -- [ ] **Step 2: Verify `uv.lock` is already pinned to 0.14.9** - -```bash -grep -A 2 '^name = "pyspiral"' uv.lock -``` - -Expected output (version must be `0.14.9`): - -``` -name = "pyspiral" -version = "0.14.9" -source = { registry = "https://pypi.org/simple" } -``` - -If the version is still `0.11.7`, run: - -```bash -uv lock --upgrade-package pyspiral -``` - -Then re-run the grep to confirm `0.14.9`. - -- [ ] **Step 3: Re-sync the environment to confirm the constraint resolves cleanly** - -```bash -uv sync --all-extras --dev -``` - -Expected: exits 0 with no resolution errors. The last few lines should show -`pyspiral-0.14.9` installed (or already satisfied). - -- [ ] **Step 4: Run the full unit test suite to confirm no regressions** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py -v -``` - -Expected: all tests PASS. The spiraldb connector unit tests mock the `spiral` -module entirely, so the version bump does not affect them. - -- [ ] **Step 5: Commit** - -```bash -git add pyproject.toml uv.lock -git commit -m "chore(deps): upgrade pyspiral 0.11.7 → 0.14.9 (PLT-1773) - -t3.storage.dev began enforcing that all HTTP request headers must appear in -X-Amz-SignedHeaders around 2026-06-15. pyspiral 0.11.7's embedded Rust HTTP -client sent additional unsigned headers, causing every Vortex file read to -return AccessDenied. pyspiral 0.14.x rewrote the HTTP stack using Python-level -httpx, which resolves the issue. - -Bumps the spiraldb extra floor from >=0.11.0 to >=0.14.0 to document the -effective minimum. No connector code changes required — full API compatibility -verified across 0.11.7 → 0.14.9 (see PLT-1773 spec). - -Closes PLT-1773" -``` - ---- - -### Task 2: Add breadcrumb to integration test module docstring - -**Files:** -- Modify: `tests/test_databases/test_spiraldb_connector_integration.py:1-19` - -- [ ] **Step 1: Extend the module docstring** - -Replace the existing module docstring (lines 1–19) with: - -```python -"""Integration tests for SpiralDBConnector against the live dev project. - -These tests are skipped unless the ``SPIRAL_INTEGRATION_TESTS=1`` env var is set. -They require a valid auth mechanism, either: - -- Local dev: ``~/.config/pyspiral/auth.json`` (obtained via ``spiral login``) -- CI (GitHub Actions): ``SPIRAL_WORKLOAD_ID`` env var pointing to a workload - that has editor access on the dev project and is bound to this repository via - GitHub OIDC (see internal CI/runbook docs for the exact workload/policy IDs). - -If the env var is set but credentials are absent, the tests fail rather than -skip — the operator is expected to ensure auth is in place when enabling -integration tests. - -Project and server URL are configurable via env vars: - -- ``SPIRAL_PROJECT_ID`` (default: ``test-orcapod-362211``) -- ``SPIRAL_SERVER_URL`` (default: ``http://api.spiraldb.dev``) - -Version compatibility note (PLT-1773) --------------------------------------- -pyspiral is in active development and must be kept current. The pinned version -in ``uv.lock`` should be upgraded regularly (see PLT-1785 for the tracking issue). - -History: around 2026-06-15, ``t3.storage.dev`` (SpiralDB's object-storage -backend) began strictly enforcing that every header in a presigned-URL request -must appear in ``X-Amz-SignedHeaders``. pyspiral 0.11.7's embedded Rust HTTP -client sent additional unsigned headers, causing all Vortex file reads to fail -with ``AccessDenied: There were headers present in the request which were not -signed``. Upgrading to pyspiral 0.14.x (which rewrote the HTTP stack in Python -using ``httpx``) resolved the issue. If these tests start failing with a similar -``AccessDenied`` / unsigned-headers error, upgrade pyspiral first. -""" -``` - -- [ ] **Step 2: Verify the file still imports cleanly** - -```bash -uv run python -c "import tests.test_databases.test_spiraldb_connector_integration" 2>&1 || \ -uv run python -c " -import importlib.util, sys -spec = importlib.util.spec_from_file_location( - 'test_mod', - 'tests/test_databases/test_spiraldb_connector_integration.py' -) -mod = importlib.util.module_from_spec(spec) -spec.loader.exec_module(mod) -print('OK') -" -``` - -Expected: `OK` (or a clean exit with no ImportError; the `pytest.mark.skipif` -at module level is fine to execute). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_databases/test_spiraldb_connector_integration.py -git commit -m "docs(spiraldb): add PLT-1773 breadcrumb to integration test module docstring - -Explains the t3.storage.dev header-signing enforcement change (2026-06-15) -and the unsigned-headers AccessDenied failure pattern so future maintainers -know to check the pyspiral version first." -``` - ---- - -### Task 3: Add entry to `DESIGN_ISSUES.md` - -**Files:** -- Modify: `DESIGN_ISSUES.md` (append new section) - -- [ ] **Step 1: Add the new entry** - -Append the following block at the end of `DESIGN_ISSUES.md` (after the last `---` -separator or at the very end of the file): - -```markdown ---- - -## `pyspiral` dependency (SpiralDB integration) - -### SP1 — pyspiral 0.11.7 broke against t3.storage.dev header-signing enforcement change -**Status:** resolved -**Severity:** high -**Issue:** PLT-1773 - -Around 2026-06-15, SpiralDB's object-storage backend (`t3.storage.dev`) began -strictly enforcing that every header present in a presigned S3-style GET request -must appear in `X-Amz-SignedHeaders`. pyspiral 0.11.7's embedded Rust HTTP -client sent additional unsigned headers alongside the presigned URL, causing all -Vortex file reads to return: - -``` -AccessDenied: There were headers present in the request which were not signed -``` - -All three `TestSpiralDBConnectorIntegration` tests failed on every CI push from -2026-06-15 onward. The runner image version was identical between the last green -and first red run, confirming this was a server-side enforcement change rather -than a runner regression. - -**Fix:** Upgraded pyspiral from `0.11.7` to `0.14.9`. The 0.14.x line rewrote -the HTTP/auth stack to use Python-level `httpx` instead of the embedded Rust -client, eliminating the unsigned-header problem. The `pyproject.toml` minimum -was bumped from `>=0.11.0` to `>=0.14.0` to document the effective floor. -No connector code changes were needed — the public API is fully compatible -across this version range. - -**Ongoing:** pyspiral releases frequently. See PLT-1785 for the tracking issue -covering routine version bumps. -``` - -- [ ] **Step 2: Run a quick smoke test to confirm no other tests broke** - -```bash -uv run pytest tests/test_databases/ -v --ignore=tests/test_databases/test_spiraldb_connector_integration.py -``` - -Expected: all tests PASS (the integration tests are excluded since they require -live SpiralDB credentials). - -- [ ] **Step 3: Commit** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs(design-issues): log SP1 — pyspiral header-signing regression (PLT-1773) - -Records the root cause (t3.storage.dev enforcement change), symptom -(AccessDenied on all Vortex reads), and resolution (pyspiral upgrade to -0.14.9) in DESIGN_ISSUES.md for future reference." -``` - ---- - -### Task 4: Push branch and verify CI - -**Files:** none - -- [ ] **Step 1: Push the branch** - -```bash -git push -u origin eywalker/plt-1773-fix-ongoing-spiral-database-test-failure-in-ci -``` - -Expected: branch pushed without errors. - -- [ ] **Step 2: Open a PR targeting `dev`** - -```bash -gh pr create \ - --base dev \ - --title "fix(spiraldb): upgrade pyspiral 0.11.7 → 0.14.9 to fix CI header-signing failure (PLT-1773)" \ - --body "$(cat <<'EOF' -## Summary - -- Upgrades `pyspiral` from `0.11.7` to `0.14.9` in `uv.lock` -- Bumps the `spiraldb` extra minimum in `pyproject.toml` from `>=0.11.0` to `>=0.14.0` -- Adds breadcrumb documentation to the integration test module docstring and `DESIGN_ISSUES.md` - -## Root cause - -Around 2026-06-15, `t3.storage.dev` (SpiralDB's object-storage backend) began strictly -enforcing that every header in a presigned-URL request must appear in -`X-Amz-SignedHeaders`. `pyspiral 0.11.7`'s embedded Rust HTTP client sent additional -unsigned headers, causing all Vortex file reads to fail with: - -``` -AccessDenied: There were headers present in the request which were not signed -``` - -`pyspiral 0.14.x` rewrites the HTTP stack using Python-level `httpx`, resolving the issue. -Full API compatibility was verified — no connector code changes were needed. - -## Test plan - -- [ ] Local: `uv run pytest tests/test_databases/test_spiraldb_connector.py -v` — all PASS -- [ ] CI: `spiral-integration` job on this push must go green (definitive end-to-end verification) - -Fixes PLT-1773 -Related: PLT-1785 (ongoing pyspiral version tracking) - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - -- [ ] **Step 3: Monitor the `spiral-integration` CI job** - -The `spiral-integration` job runs on push (not on PRs), so it will fire when the -branch is pushed. Watch the Actions tab for the run. Expected outcome: all three -`TestSpiralDBConnectorIntegration` tests PASS. - -If they fail with a different error (not the unsigned-headers `AccessDenied`), the -pyspiral 0.14.9 API may have a breaking change not caught by the static inspection. -In that case, read the new error, identify which API call broke, and update -`spiraldb_connector.py` accordingly. diff --git a/superpowers/plans/2026-06-24-plt-1660-hard-cut-extension-type-hashing.md b/superpowers/plans/2026-06-24-plt-1660-hard-cut-extension-type-hashing.md deleted file mode 100644 index a4642fdb4..000000000 --- a/superpowers/plans/2026-06-24-plt-1660-hard-cut-extension-type-hashing.md +++ /dev/null @@ -1,2466 +0,0 @@ -# PLT-1660: Hard Cut Extension Type Hashing — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Delete the old shape-based `SemanticTypeRegistry` system, wire the new extension-type system into Arrow hashing, and rename all protocol/registry/handler classes to cleaner names. - -**Architecture:** `ArrowTypeDataVisitor` gains a `visit_extension()` hook (default: passthrough). `SemanticHashingVisitor` overrides it: for extension types whose Python counterpart has a registered semantic hasher, it converts the value to a Python object, hashes it, and stores the result as `pa.large_binary()` in the format `:::`. Unrecognized extension types pass through unmodified — starfix still sees their full metadata. All `TypeHandlerProtocol.handle()->Any` handlers are tightened to `PythonTypeSemanticHasherProtocol.hash()->ContentHash`. - -**Tech Stack:** Python 3.10+, PyArrow extension types, starfix-python, uv/pytest - ---- - -## File Map - -**Modified source:** -- `src/orcapod/protocols/hashing_protocols.py` — rename `TypeHandlerProtocol`→`PythonTypeSemanticHasherProtocol`, `handle()`→`hash()->ContentHash`; rename `type_handler_registry`→`type_semantic_hasher_registry` on `SemanticHasherProtocol` -- `src/orcapod/hashing/semantic_hashing/type_handler_registry.py` — rename class + all methods -- `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` — rename 11 handler classes; `handle()`→`hash()->ContentHash`; rename `register_builtin_handlers` -- `src/orcapod/hashing/semantic_hashing/semantic_hasher.py` — rename `BaseSemanticHasher`→`SemanticAwarePythonHasher`; simplify dispatch; rename property -- `src/orcapod/hashing/semantic_hashing/content_identifiable_mixin.py` — update import + type annotations -- `src/orcapod/hashing/semantic_hashing/__init__.py` — update exports -- `src/orcapod/hashing/__init__.py` — update exports -- `src/orcapod/hashing/defaults.py` — rename function; update property access; remove broken `set_cacher` call -- `src/orcapod/hashing/visitors.py` — add `visit_extension` to base class + rewrite `SemanticHashingVisitor` -- `src/orcapod/hashing/arrow_hashers.py` — update `StarfixArrowHasher` constructor + short-circuit; delete `SemanticArrowHasher` -- `src/orcapod/hashing/versioned_hashers.py` — source `StarfixArrowHasher` from context; rename imports -- `src/orcapod/contexts/data/v0.1.json` — reorder components; remove `semantic_registry`; update class names and refs; add `type_converter`+`semantic_hasher` to `arrow_hasher`; remove `pa.Table` handlers (cycle-break) -- `src/orcapod/contexts/data/schemas/context_schema.json` — remove `semantic_registry` property; rename `type_handler_registry`→`python_type_semantic_hasher_registry` -- `src/orcapod/contexts/core.py` — update docstring for renamed property -- `src/orcapod/semantic_types/__init__.py` — remove `SemanticTypeRegistry` export -- `src/orcapod/protocols/semantic_types_protocols.py` — delete `SemanticStructConverterProtocol` - -**Deleted source:** -- `src/orcapod/semantic_types/semantic_struct_converters.py` -- `src/orcapod/semantic_types/semantic_registry.py` - -**Deleted tests:** -- `tests/test_semantic_types/` (all 9 files) -- `tests/test_hashing/test_file_hashing_consistency.py` - -**New tests:** -- `tests/test_hashing/test_extension_type_hashing.py` - -**Updated tests:** -- `tests/test_hashing/test_semantic_hasher.py` -- `tests/test_hashing/test_starfix_arrow_hasher.py` - ---- - -## Task 1: Rename `TypeHandlerProtocol` → `PythonTypeSemanticHasherProtocol` - -**Files:** -- Modify: `src/orcapod/protocols/hashing_protocols.py` - -- [ ] **Step 1: Rewrite the protocol class and update surrounding references** - -Replace the entire `TypeHandlerProtocol` class and update the `SemanticHasherProtocol`'s `type_handler_registry` property: - -```python -# In src/orcapod/protocols/hashing_protocols.py - -# Update TYPE_CHECKING import: -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeSemanticHasherRegistry - from orcapod.types import ContentHash # already imported at module level, just noting - -# Replace TypeHandlerProtocol with: -class PythonTypeSemanticHasherProtocol(Protocol): - """Protocol for type-specific semantic hashers used by SemanticAwarePythonHasher. - - A PythonTypeSemanticHasherProtocol hashes a specific Python type to a ``ContentHash``. - Implementations are registered with a ``PythonTypeSemanticHasherRegistry`` and looked - up via MRO-aware resolution. - - Each implementation receives the full ``SemanticAwarePythonHasher`` so it can delegate - hashing of sub-values (e.g. hashing a dict of function metadata) back to the outer - hasher without coupling to a specific hasher instance. - """ - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - """Hash *obj* to a ContentHash. - - Args: - obj: The object to hash. Always matches the registered type. - hasher: The active ``SemanticAwarePythonHasher``. Use - ``hasher.hash_object(sub_value)`` to hash sub-values. - - Returns: - ContentHash: The content-addressed hash of *obj*. - """ - ... - - -# Update SemanticHasherProtocol — rename the property: -class SemanticHasherProtocol(Protocol): - # ... existing methods unchanged ... - - @property - def type_semantic_hasher_registry(self) -> "PythonTypeSemanticHasherRegistry": - """Return the PythonTypeSemanticHasherRegistry used by this hasher.""" - ... -``` - -The full updated `hashing_protocols.py` (only `TypeHandlerProtocol` is renamed and `SemanticHasherProtocol.type_handler_registry` → `type_semantic_hasher_registry`; everything else is unchanged): - -```python -"""Hash strategy protocols for dependency injection.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable - -from orcapod.types import ContentHash, PathLike, Schema - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeSemanticHasherRegistry - - -@runtime_checkable -class DataContextAwareProtocol(Protocol): - """Protocol for objects aware of their data context.""" - - @property - def data_context_key(self) -> str: - """Return the data context key associated with this object.""" - ... - - -@runtime_checkable -class PipelineElementProtocol(Protocol): - """Protocol for objects that have a stable identity as an element in a pipeline graph.""" - - def pipeline_identity_structure(self) -> Any: - """Return a structure representing this element's pipeline identity.""" - ... - - def pipeline_hash(self, hasher=None) -> ContentHash: - """Return the pipeline-level hash of this element.""" - ... - - -@runtime_checkable -class ContentIdentifiableProtocol(Protocol): - """Protocol for objects that can express their semantic identity as a plain Python structure.""" - - def identity_structure(self) -> Any: - """Return a structure that represents the semantic identity of this object.""" - ... - - def content_hash(self, hasher: "SemanticHasherProtocol | None" = None) -> ContentHash: - """Returns the content hash.""" - ... - - -class PythonTypeSemanticHasherProtocol(Protocol): - """Protocol for type-specific semantic hashers used by SemanticAwarePythonHasher. - - A ``PythonTypeSemanticHasherProtocol`` hashes a specific Python type to a - ``ContentHash``. Implementations are registered with a - ``PythonTypeSemanticHasherRegistry`` and looked up via MRO-aware resolution. - - Each implementation receives the full ``SemanticAwarePythonHasher`` so it can - delegate hashing of sub-values back to the outer hasher without coupling to a - specific hasher instance. - """ - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - """Hash *obj* to a ContentHash. - - Args: - obj: The object to hash. Always matches the registered type. - hasher: The active ``SemanticAwarePythonHasher``. Use - ``hasher.hash_object(sub_value)`` to hash sub-values. - - Returns: - ContentHash: The content-addressed hash of *obj*. - """ - ... - - -class SemanticHasherProtocol(Protocol): - """Protocol for the semantic content-based hasher.""" - - def hash_object( - self, - obj: Any, - resolver: Callable[[Any], ContentHash] | None = None, - ) -> ContentHash: - """Hash *obj* based on its semantic content.""" - ... - - @property - def hasher_id(self) -> str: - """Returns a unique identifier/name for this hasher instance.""" - ... - - @property - def type_semantic_hasher_registry(self) -> "PythonTypeSemanticHasherRegistry": - """Return the PythonTypeSemanticHasherRegistry used by this hasher.""" - ... - - -class FileContentHasherProtocol(Protocol): - """Protocol for file-related hashing.""" - - def hash_file(self, file_path: PathLike) -> ContentHash: ... - - -@runtime_checkable -class ArrowHasherProtocol(Protocol): - """Protocol for hashing arrow data.""" - - @property - def hasher_id(self) -> str: ... - - def hash_table(self, table: "pa.Table | pa.RecordBatch") -> ContentHash: ... - - -class StringCacherProtocol(Protocol): - """Protocol for caching string key value pairs.""" - - def get_cached(self, cache_key: str) -> str | None: ... - def set_cached(self, cache_key: str, value: str) -> None: ... - def clear_cache(self) -> None: ... - - -class FunctionInfoExtractorProtocol(Protocol): - """Protocol for extracting function information.""" - - def extract_function_info( - self, - func: Callable[..., Any], - function_name: str | None = None, - input_typespec: Schema | None = None, - output_typespec: Schema | None = None, - exclude_function_signature: bool = False, - exclude_function_body: bool = False, - ) -> dict[str, Any]: ... - - -class SemanticTypeHasherProtocol(Protocol): - """Abstract base class for semantic type-specific hashers.""" - - @property - def hasher_id(self) -> str: - """Unique identifier for this semantic type hasher.""" - ... - - def hash_column(self, column: "pa.Array") -> "pa.Array": - """Hash a column with this semantic type and return the hash bytes as an array.""" - ... - - def set_cacher(self, cacher: StringCacherProtocol) -> None: - """Add a string cacher for caching hash values.""" - ... -``` - -- [ ] **Step 2: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/protocols/hashing_protocols.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor(hashing_protocols): rename TypeHandlerProtocol → PythonTypeSemanticHasherProtocol, tighten hash() → ContentHash" -``` - ---- - -## Task 2: Rename `TypeHandlerRegistry` → `PythonTypeSemanticHasherRegistry` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/type_handler_registry.py` - -- [ ] **Step 1: Rename the class, subclass, and all methods** - -Write the complete new file: - -```python -""" -PythonTypeSemanticHasherRegistry — MRO-aware registry for PythonTypeSemanticHasherProtocol instances. -""" - -from __future__ import annotations - -import logging -import threading -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - PythonTypeSemanticHasherProtocol, - ) - -logger = logging.getLogger(__name__) - - -class PythonTypeSemanticHasherRegistry: - """Registry mapping Python types to PythonTypeSemanticHasherProtocol instances. - - Lookup is MRO-aware: when no hasher is registered for the exact type of - an object, the registry walks the object's MRO (most-derived first) until - it finds a match. - - Thread safety - ------------- - Registration and lookup are protected by a reentrant lock so that the - global singleton can be safely used from multiple threads. - """ - - def __init__( - self, handlers: list[tuple[type, "PythonTypeSemanticHasherProtocol"]] | None = None - ) -> None: - """ - Args: - handlers: Optional list of ``(target_type, hasher)`` pairs to - register at construction time. - """ - self._handlers: dict[type, "PythonTypeSemanticHasherProtocol"] = {} - self._lock = threading.RLock() - if handlers: - for target_type, handler in handlers: - self.register(target_type, handler) - - def register(self, target_type: type, handler: "PythonTypeSemanticHasherProtocol") -> None: - """Register a hasher for a specific Python type. - - If a hasher is already registered for *target_type*, it is silently - replaced by the new hasher. - - Args: - target_type: The Python type (or class) for which the hasher should be used. - handler: A ``PythonTypeSemanticHasherProtocol`` instance. - - Raises: - TypeError: If ``target_type`` is not a ``type``. - """ - if not isinstance(target_type, type): - raise TypeError( - f"target_type must be a type/class, got {type(target_type)!r}" - ) - with self._lock: - existing = self._handlers.get(target_type) - if existing is not None and existing is not handler: - logger.debug( - "PythonTypeSemanticHasherRegistry: replacing existing hasher for %s (%s -> %s)", - target_type.__name__, - type(existing).__name__, - type(handler).__name__, - ) - self._handlers[target_type] = handler - - def unregister(self, target_type: type) -> bool: - """Remove the hasher registered for *target_type*, if any. - - Args: - target_type: The type whose hasher should be removed. - - Returns: - True if a hasher was removed, False if none was registered. - """ - with self._lock: - if target_type in self._handlers: - del self._handlers[target_type] - return True - return False - - def get_semantic_hasher(self, obj: Any) -> "PythonTypeSemanticHasherProtocol | None": - """Look up the hasher for *obj* using MRO-aware resolution. - - Args: - obj: The object for which a hasher is needed. - - Returns: - The registered ``PythonTypeSemanticHasherProtocol``, or None. - """ - obj_type = type(obj) - with self._lock: - handler = self._handlers.get(obj_type) - if handler is not None: - return handler - for base in obj_type.__mro__[1:]: - handler = self._handlers.get(base) - if handler is not None: - logger.debug( - "PythonTypeSemanticHasherRegistry: resolved hasher for %s via base %s", - obj_type.__name__, - base.__name__, - ) - return handler - return None - - def get_semantic_hasher_for_type( - self, target_type: type - ) -> "PythonTypeSemanticHasherProtocol | None": - """Look up the hasher for a *type object* (rather than an instance). - - Args: - target_type: The type to look up. - - Returns: - The registered ``PythonTypeSemanticHasherProtocol``, or None. - """ - with self._lock: - handler = self._handlers.get(target_type) - if handler is not None: - return handler - for base in target_type.__mro__[1:]: - handler = self._handlers.get(base) - if handler is not None: - return handler - return None - - def has_semantic_hasher(self, target_type: type) -> bool: - """Return True if a hasher is registered for *target_type* or any MRO ancestor. - - Args: - target_type: The type to check. - """ - return self.get_semantic_hasher_for_type(target_type) is not None - - def registered_types(self) -> list[type]: - """Return a list of all directly-registered types (no MRO expansion).""" - with self._lock: - return list(self._handlers.keys()) - - def __repr__(self) -> str: - with self._lock: - names = [t.__name__ for t in self._handlers] - return f"PythonTypeSemanticHasherRegistry(registered={names!r})" - - def __len__(self) -> int: - with self._lock: - return len(self._handlers) - - -def get_default_python_type_semantic_hasher_registry() -> "PythonTypeSemanticHasherRegistry": - """Return the PythonTypeSemanticHasherRegistry from the default data context. - - This is a convenience wrapper; the registry is owned and versioned by the - active ``DataContext``. Importing this function from - ``orcapod.hashing.defaults`` or ``orcapod.hashing`` is equivalent. - """ - from orcapod.hashing.defaults import ( - get_default_python_type_semantic_hasher_registry as _get, - ) - return _get() - - -class BuiltinPythonTypeSemanticHasherRegistry(PythonTypeSemanticHasherRegistry): - """A PythonTypeSemanticHasherRegistry pre-populated with all built-in hashers. - - Constructed via the data context JSON spec so that the default registry - is versioned alongside the rest of the context components. - """ - - def __init__(self, arrow_hasher: "ArrowHasherProtocol | None" = None) -> None: - super().__init__() - from orcapod.hashing.semantic_hashing.builtin_handlers import ( - register_builtin_python_type_semantic_hashers, - ) - register_builtin_python_type_semantic_hashers(self, arrow_hasher=arrow_hasher) -``` - -- [ ] **Step 2: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/semantic_hashing/type_handler_registry.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor(type_handler_registry): rename to PythonTypeSemanticHasherRegistry, rename methods" -``` - ---- - -## Task 3: Rename + tighten all builtin handlers - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` - -- [ ] **Step 1: Write the complete updated file** - -Key changes: -- 11 class renames (all `*Handler`/`*ContentHandler` → `*SemanticHasher`) -- `handle(obj, hasher) -> Any` → `hash(obj, hasher) -> ContentHash` on every class -- `UUIDSemanticHasher`, `BytesSemanticHasher`, `FunctionSemanticHasher`, `TypeObjectSemanticHasher`, `SpecialFormSemanticHasher`, `GenericAliasSemanticHasher`, `UnionTypeSemanticHasher` now call `hasher.hash_object(...)` to return `ContentHash` directly -- `register_builtin_handlers` → `register_builtin_python_type_semantic_hashers` -- Remove `SemanticArrowHasher` fallback construction (it will be deleted); when `arrow_hasher is None`, skip registering `pa.Table`/`pa.RecordBatch` handlers - -```python -""" -Built-in PythonTypeSemanticHasherProtocol implementations. - - PathSemanticHasher -- pathlib.Path: file content hash - UPathSemanticHasher -- upath.UPath: file content hash (remote-aware) - UUIDSemanticHasher -- uuid.UUID: 16-byte binary representation - BytesSemanticHasher -- bytes/bytearray: hex string representation - FunctionSemanticHasher -- callable with __code__: via FunctionInfoExtractorProtocol - TypeObjectSemanticHasher -- type objects: stable "type:." string - SpecialFormSemanticHasher -- typing._SpecialForm - GenericAliasSemanticHasher -- generic alias type annotations - UnionTypeSemanticHasher -- types.UnionType (Python 3.10+ X | Y syntax) - ArrowTableSemanticHasher -- pa.Table / pa.RecordBatch - SchemaSemanticHasher -- Schema objects - -``register_builtin_python_type_semantic_hashers(registry)`` populates a registry -with all of the above. -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import TYPE_CHECKING, Any -from uuid import UUID - -from upath import UPath - -from orcapod.types import ContentHash, PathLike, Schema - -if TYPE_CHECKING: - from orcapod.hashing.semantic_hashing.type_handler_registry import ( - PythonTypeSemanticHasherRegistry, - ) - from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher - from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - FileContentHasherProtocol, - ) - -logger = logging.getLogger(__name__) - - -class PathSemanticHasher: - """Hasher for pathlib.Path objects — hashes file *content*. - - Args: - file_hasher: Any object with a ``hash_file(path) -> ContentHash`` method. - """ - - def __init__(self, file_hasher: "FileContentHasherProtocol") -> None: - self.file_hasher = file_hasher - - def hash(self, obj: PathLike, hasher: "SemanticAwarePythonHasher") -> ContentHash: - path: Path = Path(obj) - if not path.exists(): - raise FileNotFoundError( - f"PathSemanticHasher: path does not exist: {path!r}. " - "Paths must refer to existing files for content-based hashing." - ) - if path.is_dir(): - raise IsADirectoryError( - f"PathSemanticHasher: path is a directory: {path!r}. " - "Only regular files are supported for content-based hashing." - ) - logger.debug("PathSemanticHasher: hashing file content at %s", path) - return self.file_hasher.hash_file(path) - - -class UPathSemanticHasher: - """Hasher for universal_pathlib.UPath objects — hashes file content. - - Args: - file_hasher: Any object with a ``hash_file(path) -> ContentHash`` method. - """ - - def __init__(self, file_hasher: "FileContentHasherProtocol") -> None: - self.file_hasher = file_hasher - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - if not isinstance(obj, UPath): - raise TypeError( - f"UPathSemanticHasher: expected a UPath, got {type(obj)!r}." - ) - if not obj.exists(): - raise FileNotFoundError( - f"UPathSemanticHasher: path does not exist: {obj!r}." - ) - if obj.is_dir(): - raise IsADirectoryError( - f"UPathSemanticHasher: path is a directory: {obj!r}." - ) - logger.debug("UPathSemanticHasher: hashing file content at %s", obj) - return self.file_hasher.hash_file(obj) - - -class UUIDSemanticHasher: - """Hasher for ``uuid.UUID`` objects — hashes the raw 16-byte binary representation.""" - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - return hasher.hash_object(obj.bytes) - - -class BytesSemanticHasher: - """Hasher for bytes and bytearray objects — hashes the lowercase hex representation.""" - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - if isinstance(obj, (bytes, bytearray)): - return hasher.hash_object(obj.hex()) - raise TypeError( - f"BytesSemanticHasher: expected bytes or bytearray, got {type(obj)!r}" - ) - - -class FunctionSemanticHasher: - """Hasher for Python functions/callables with a ``__code__`` attribute. - - Args: - function_info_extractor: Any object with an - ``extract_function_info(func) -> dict`` method. - """ - - def __init__(self, function_info_extractor: Any) -> None: - self.function_info_extractor = function_info_extractor - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - if not (callable(obj) and hasattr(obj, "__code__")): - raise TypeError( - f"FunctionSemanticHasher: expected a callable with __code__, got {type(obj)!r}" - ) - func_name = getattr(obj, "__name__", repr(obj)) - logger.debug("FunctionSemanticHasher: extracting info for function %r", func_name) - info: dict[str, Any] = self.function_info_extractor.extract_function_info(obj) - return hasher.hash_object(info) - - -class TypeObjectSemanticHasher: - """Hasher for type objects (classes passed as values). - - Returns a stable string of the form ``"type:."``. - """ - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - if not isinstance(obj, type): - raise TypeError( - f"TypeObjectSemanticHasher: expected a type/class, got {type(obj)!r}" - ) - module: str = obj.__module__ or "" - qualname: str = obj.__qualname__ - return hasher.hash_object(f"type:{module}.{qualname}") - - -class SpecialFormSemanticHasher: - """Hasher for ``typing._SpecialForm`` objects such as ``typing.Union``.""" - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - name = getattr(obj, "_name", None) or repr(obj) - return hasher.hash_object(f"special_form:typing.{name}") - - -class GenericAliasSemanticHasher: - """Hasher for generic alias type annotations (``dict[int, str]``, ``Optional[X]``, etc.).""" - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - import typing - - origin = getattr(obj, "__origin__", None) - args = getattr(obj, "__args__", None) or () - if origin is None: - return hasher.hash_object(f"generic_alias:{obj!r}") - if origin is typing.Union: - hashed_args = sorted(hasher.hash_object(arg).to_string() for arg in args) - return hasher.hash_object({"__type__": "union", "args": hashed_args}) - return hasher.hash_object({ - "__type__": "generic_alias", - "origin": hasher.hash_object(origin).to_string(), - "args": [hasher.hash_object(arg).to_string() for arg in args], - }) - - -class UnionTypeSemanticHasher: - """Hasher for ``types.UnionType`` objects (Python 3.10+ ``X | Y`` syntax).""" - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - args = getattr(obj, "__args__", None) or () - hashed_args = sorted(hasher.hash_object(arg).to_string() for arg in args) - return hasher.hash_object({"__type__": "union", "args": hashed_args}) - - -class ArrowTableSemanticHasher: - """Hasher for ``pa.Table`` and ``pa.RecordBatch`` objects. - - Args: - arrow_hasher: Any object satisfying ``ArrowHasherProtocol``. - """ - - def __init__(self, arrow_hasher: "ArrowHasherProtocol") -> None: - self.arrow_hasher = arrow_hasher - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - import pyarrow as _pa - - if isinstance(obj, _pa.RecordBatch): - obj = _pa.Table.from_batches([obj]) - if not isinstance(obj, _pa.Table): - raise TypeError( - f"ArrowTableSemanticHasher: expected pa.Table or pa.RecordBatch, got {type(obj)!r}" - ) - return self.arrow_hasher.hash_table(obj) - - -class SchemaSemanticHasher: - """Hasher for ``Schema`` objects.""" - - def hash(self, obj: Any, hasher: "SemanticAwarePythonHasher") -> ContentHash: - if not isinstance(obj, Schema): - raise TypeError( - f"SchemaSemanticHasher: expected a Schema, got {type(obj)!r}" - ) - raise NotImplementedError("SchemaSemanticHasher is not yet implemented.") - - -def register_builtin_python_type_semantic_hashers( - registry: "PythonTypeSemanticHasherRegistry", - file_hasher: Any = None, - function_info_extractor: Any = None, - arrow_hasher: "ArrowHasherProtocol | None" = None, -) -> None: - """Register all built-in semantic hashers into *registry*. - - When ``arrow_hasher`` is None, ``pa.Table`` and ``pa.RecordBatch`` handlers - are **not** registered (to avoid circular dependency in the JSON context - construction — the default context's ``python_type_semantic_hasher_registry`` - is built before ``arrow_hasher``). - - Args: - registry: The ``PythonTypeSemanticHasherRegistry`` to populate. - file_hasher: Optional ``FileContentHasherProtocol`` for path hashing. - Defaults to ``BasicFileHasher(sha256)``. - function_info_extractor: Optional ``FunctionInfoExtractorProtocol``. - Defaults to ``FunctionSignatureExtractor``. - arrow_hasher: Optional ``ArrowHasherProtocol`` for nested table hashing. - When None, Arrow table handlers are skipped. - """ - if file_hasher is None: - from orcapod.hashing.file_hashers import BasicFileHasher - file_hasher = BasicFileHasher(algorithm="sha256") - - if function_info_extractor is None: - from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionSignatureExtractor, - ) - function_info_extractor = FunctionSignatureExtractor( - include_module=True, - include_defaults=True, - ) - - bytes_hasher = BytesSemanticHasher() - registry.register(bytes, bytes_hasher) - registry.register(bytearray, bytes_hasher) - - registry.register(Path, PathSemanticHasher(file_hasher)) - registry.register(UPath, UPathSemanticHasher(file_hasher)) - registry.register(UUID, UUIDSemanticHasher()) - - import types as _types - - function_hasher = FunctionSemanticHasher(function_info_extractor) - registry.register(_types.FunctionType, function_hasher) - registry.register(_types.BuiltinFunctionType, function_hasher) - registry.register(_types.MethodType, function_hasher) - - registry.register(type, TypeObjectSemanticHasher()) - registry.register(_types.UnionType, UnionTypeSemanticHasher()) - - generic_alias_hasher = GenericAliasSemanticHasher() - registry.register(_types.GenericAlias, generic_alias_hasher) - try: - import typing as _typing - registry.register(_typing._GenericAlias, generic_alias_hasher) # type: ignore[attr-defined] - registry.register(_typing._SpecialForm, SpecialFormSemanticHasher()) # type: ignore[attr-defined] - except AttributeError: - pass - - registry.register(Schema, SchemaSemanticHasher()) - - if arrow_hasher is not None: - import pyarrow as _pa - arrow_table_hasher = ArrowTableSemanticHasher(arrow_hasher) - registry.register(_pa.Table, arrow_table_hasher) - registry.register(_pa.RecordBatch, arrow_table_hasher) - - logger.debug( - "register_builtin_python_type_semantic_hashers: registered %d hashers", - len(registry), - ) -``` - -- [ ] **Step 2: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/semantic_hashing/builtin_handlers.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor(builtin_handlers): rename handler classes, tighten hash() → ContentHash" -``` - ---- - -## Task 4: Rename `BaseSemanticHasher` → `SemanticAwarePythonHasher`, simplify dispatch - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/semantic_hasher.py` - -- [ ] **Step 1: Apply renames and simplify hash_object dispatch** - -Changes: -1. Class name `BaseSemanticHasher` → `SemanticAwarePythonHasher` -2. `__init__` parameter `type_handler_registry` → `type_semantic_hasher_registry` -3. `self._registry = get_default_type_handler_registry()` → `get_default_python_type_semantic_hasher_registry()` -4. `type_handler_registry` property → `type_semantic_hasher_registry` -5. Return type annotation `TypeHandlerRegistry` → `PythonTypeSemanticHasherRegistry` -6. `hash_object` dispatch: `get_handler` → `get_semantic_hasher`; remove double-wrap (handler now returns `ContentHash` directly) - -The dispatch block in `hash_object` changes from: -```python -handler = self._registry.get_handler(obj) -if handler is not None: - return self.hash_object(handler.handle(obj, self), resolver=resolver) -``` -to: -```python -semantic_hasher = self._registry.get_semantic_hasher(obj) -if semantic_hasher is not None: - return semantic_hasher.hash(obj, self) -``` - -Full updated file (only showing the changed parts — keep everything else identical): - -```python -# At top of file, update import: -from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeSemanticHasherRegistry - -# Class rename: -class SemanticAwarePythonHasher: - """ - Content-based recursive hasher. - [same docstring, just update BaseSemanticHasher references to SemanticAwarePythonHasher] - """ - - def __init__( - self, - hasher_id: str, - type_semantic_hasher_registry: PythonTypeSemanticHasherRegistry | None = None, - strict: bool = True, - ) -> None: - self._hasher_id = hasher_id - self._strict = strict - - if type_semantic_hasher_registry is None: - from orcapod.hashing.defaults import get_default_python_type_semantic_hasher_registry - self._registry = get_default_python_type_semantic_hasher_registry() - else: - self._registry = type_semantic_hasher_registry - - @property - def hasher_id(self) -> str: - return self._hasher_id - - @property - def strict(self) -> bool: - return self._strict - - @property - def type_semantic_hasher_registry(self) -> PythonTypeSemanticHasherRegistry: - """Return the ``PythonTypeSemanticHasherRegistry`` used by this hasher.""" - return self._registry - - def hash_object(self, obj, resolver=None): - # ... keep all existing logic, EXCEPT replace the handler dispatch block: - - # Old: - # handler = self._registry.get_handler(obj) - # if handler is not None: - # return self.hash_object(handler.handle(obj, self), resolver=resolver) - - # New: - # semantic_hasher = self._registry.get_semantic_hasher(obj) - # if semantic_hasher is not None: - # return semantic_hasher.hash(obj, self) - ... -``` - -The complete updated `hash_object` method (copy the full existing body, changing only the handler dispatch): - -```python -def hash_object( - self, - obj: Any, - resolver: Callable[[Any], ContentHash] | None = None, -) -> ContentHash: - """Hash *obj* based on its semantic content.""" - # Terminal: already a hash -- return as-is. - if isinstance(obj, ContentHash): - return obj - - # Primitives: hash their direct JSON representation. - if isinstance(obj, (type(None), bool, int, float, str)): - return self._hash_to_content_hash(obj) - - # Structures: expand into a tagged tree, then hash the tree. - if _is_structure(obj): - expanded = self._expand_structure( - obj, _visited=frozenset(), resolver=resolver - ) - return self._hash_to_content_hash(expanded) - - # Semantic hasher dispatch: the hasher produces a ContentHash directly. - semantic_hasher = self._registry.get_semantic_hasher(obj) - if semantic_hasher is not None: - logger.debug( - "hash_object: dispatching %s to semantic hasher %s", - type(obj).__name__, - type(semantic_hasher).__name__, - ) - return semantic_hasher.hash(obj, self) - - # ContentIdentifiableProtocol: use resolver if provided, else content_hash(). - if isinstance(obj, hp.ContentIdentifiableProtocol): - if resolver is not None: - logger.debug( - "hash_object: resolving ContentIdentifiableProtocol %s via resolver", - type(obj).__name__, - ) - return resolver(obj) - else: - logger.debug( - "hash_object: using ContentIdentifiableProtocol %s's content_hash", - type(obj).__name__, - ) - return obj.content_hash() - - # Fallback for unhandled types. - fallback = self._handle_unknown(obj) - return self._hash_to_content_hash(fallback) -``` - -- [ ] **Step 2: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/semantic_hashing/semantic_hasher.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor(semantic_hasher): rename BaseSemanticHasher → SemanticAwarePythonHasher, simplify dispatch" -``` - ---- - -## Task 5: Update `content_identifiable_mixin.py` and `contexts/core.py` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/content_identifiable_mixin.py` -- Modify: `src/orcapod/contexts/core.py` - -- [ ] **Step 1: Update `content_identifiable_mixin.py`** - -Three changes: -1. Line 68: `from orcapod.hashing.semantic_hashing.semantic_hasher import BaseSemanticHasher` → `SemanticAwarePythonHasher` -2. Line 97: parameter `semantic_hasher: BaseSemanticHasher | None` → `SemanticAwarePythonHasher | None` -3. Line 218 (approximately): `def _get_hasher(self) -> BaseSemanticHasher:` → `SemanticAwarePythonHasher` -4. Update the class docstring reference from `BaseSemanticHasher` to `SemanticAwarePythonHasher` - -```python -# Old line 68: -from orcapod.hashing.semantic_hashing.semantic_hasher import BaseSemanticHasher - -# New: -from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher -``` - -```python -# Old __init__ signature: -def __init__( - self, *, semantic_hasher: BaseSemanticHasher | None = None, **kwargs: Any -) -> None: - -# New: -def __init__( - self, *, semantic_hasher: SemanticAwarePythonHasher | None = None, **kwargs: Any -) -> None: -``` - -Also update the `_get_hasher` return type annotation and any docstring mentions of `BaseSemanticHasher`. - -- [ ] **Step 2: Update `contexts/core.py` docstring** - -Update the `DataContext` docstring — replace `semantic_hasher.type_handler_registry` with `semantic_hasher.type_semantic_hasher_registry`: - -```python -@dataclass -class DataContext: - """Data context containing all versioned components needed for data interpretation. - - Attributes: - context_key: Unique identifier (e.g., "std:v0.1:default") - version: Version string (e.g., "v0.1") - description: Human-readable description - type_converter: Type converter for Python ↔ Arrow conversion and registration. - arrow_hasher: Arrow table hasher for this context. - semantic_hasher: General semantic hasher for this context. The - ``PythonTypeSemanticHasherRegistry`` used for hashing is accessible via - ``semantic_hasher.type_semantic_hasher_registry``. - """ -``` - -- [ ] **Step 3: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/semantic_hashing/content_identifiable_mixin.py \ - src/orcapod/contexts/core.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor: update BaseSemanticHasher → SemanticAwarePythonHasher refs in mixin and core" -``` - ---- - -## Task 6: Update `__init__.py` exports and `defaults.py` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/__init__.py` -- Modify: `src/orcapod/hashing/__init__.py` -- Modify: `src/orcapod/hashing/defaults.py` - -- [ ] **Step 1: Update `semantic_hashing/__init__.py`** - -```python -""" -orcapod.hashing.semantic_hashing -================================= - SemanticAwarePythonHasher -- content-based recursive object hasher - PythonTypeSemanticHasherRegistry -- MRO-aware registry mapping types → PythonTypeSemanticHasherProtocol - BuiltinPythonTypeSemanticHasherRegistry -- pre-populated registry with built-in hashers - ContentIdentifiableMixin -- convenience mixin for content-identifiable objects - -Built-in PythonTypeSemanticHasherProtocol implementations: - PathSemanticHasher -- pathlib.Path → file-content hash - UUIDSemanticHasher -- uuid.UUID → canonical bytes - BytesSemanticHasher -- bytes/bytearray → hex string - FunctionSemanticHasher -- callable → via FunctionInfoExtractorProtocol - TypeObjectSemanticHasher -- type objects → "type:." - register_builtin_python_type_semantic_hashers -- populate a registry with all of the above - -Function info extractors (used by FunctionSemanticHasher): - FunctionNameExtractor - FunctionSignatureExtractor - FunctionInfoExtractorFactory -""" - -from orcapod.hashing.semantic_hashing.builtin_handlers import ( - BytesSemanticHasher, - FunctionSemanticHasher, - PathSemanticHasher, - TypeObjectSemanticHasher, - UUIDSemanticHasher, - register_builtin_python_type_semantic_hashers, -) -from orcapod.hashing.semantic_hashing.content_identifiable_mixin import ( - ContentIdentifiableMixin, -) -from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionInfoExtractorFactory, - FunctionNameExtractor, - FunctionSignatureExtractor, -) -from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher -from orcapod.hashing.semantic_hashing.type_handler_registry import ( - BuiltinPythonTypeSemanticHasherRegistry, - PythonTypeSemanticHasherRegistry, -) - -__all__ = [ - "SemanticAwarePythonHasher", - "PythonTypeSemanticHasherRegistry", - "BuiltinPythonTypeSemanticHasherRegistry", - "ContentIdentifiableMixin", - "PathSemanticHasher", - "UUIDSemanticHasher", - "BytesSemanticHasher", - "FunctionSemanticHasher", - "TypeObjectSemanticHasher", - "register_builtin_python_type_semantic_hashers", - "FunctionNameExtractor", - "FunctionSignatureExtractor", - "FunctionInfoExtractorFactory", -] -``` - -- [ ] **Step 2: Update `hashing/__init__.py`** - -```python -""" -OrcaPod hashing package. - -Public API ----------- - SemanticAwarePythonHasher -- content-based recursive object hasher - SemanticHasherProtocol -- protocol for semantic hashers - PythonTypeSemanticHasherRegistry -- registry mapping types to PythonTypeSemanticHasherProtocol instances - get_default_semantic_hasher -- global default SemanticHasherProtocol factory - get_default_python_type_semantic_hasher_registry -- global default registry factory - ContentIdentifiableMixin -- convenience mixin for content-identifiable objects - -Built-in hashers (importable for custom registry setup): - PathSemanticHasher - UUIDSemanticHasher - BytesSemanticHasher - FunctionSemanticHasher - TypeObjectSemanticHasher - register_builtin_python_type_semantic_hashers - -Utility: - FileContentHasherProtocol - StringCacherProtocol - FunctionInfoExtractorProtocol - ArrowHasherProtocol -""" - -from orcapod.hashing.defaults import ( - get_default_arrow_hasher, - get_default_python_type_semantic_hasher_registry, - get_default_semantic_hasher, -) -from orcapod.hashing.file_hashers import BasicFileHasher, CachedFileHasher -from orcapod.hashing.hash_utils import hash_file -from orcapod.hashing.semantic_hashing.builtin_handlers import ( - BytesSemanticHasher, - FunctionSemanticHasher, - PathSemanticHasher, - TypeObjectSemanticHasher, - UUIDSemanticHasher, - register_builtin_python_type_semantic_hashers, -) -from orcapod.hashing.semantic_hashing.content_identifiable_mixin import ( - ContentIdentifiableMixin, -) -from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher -from orcapod.hashing.semantic_hashing.type_handler_registry import ( - BuiltinPythonTypeSemanticHasherRegistry, - PythonTypeSemanticHasherRegistry, -) -from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - ContentIdentifiableProtocol, - FileContentHasherProtocol, - FunctionInfoExtractorProtocol, - PythonTypeSemanticHasherProtocol, - SemanticHasherProtocol, - SemanticTypeHasherProtocol, - StringCacherProtocol, -) - -try: - from orcapod.hashing.legacy_core import ( - HashableMixin, - function_content_hash, - get_function_signature, - hash_function, - hash_data, - hash_pathset, - hash_to_hex, - hash_to_int, - hash_to_uuid, - ) -except ImportError: - HashableMixin = None # type: ignore[assignment,misc] - function_content_hash = None # type: ignore[assignment] - get_function_signature = None # type: ignore[assignment] - hash_function = None # type: ignore[assignment] - hash_data = None # type: ignore[assignment] - hash_pathset = None # type: ignore[assignment] - hash_to_hex = None # type: ignore[assignment] - hash_to_int = None # type: ignore[assignment] - hash_to_uuid = None # type: ignore[assignment] - -__all__ = [ - "SemanticAwarePythonHasher", - "PythonTypeSemanticHasherRegistry", - "BuiltinPythonTypeSemanticHasherRegistry", - "get_default_python_type_semantic_hasher_registry", - "get_default_semantic_hasher", - "ContentIdentifiableMixin", - "PathSemanticHasher", - "UUIDSemanticHasher", - "BytesSemanticHasher", - "FunctionSemanticHasher", - "TypeObjectSemanticHasher", - "register_builtin_python_type_semantic_hashers", - "SemanticHasherProtocol", - "ContentIdentifiableProtocol", - "PythonTypeSemanticHasherProtocol", - "FileContentHasherProtocol", - "ArrowHasherProtocol", - "StringCacherProtocol", - "FunctionInfoExtractorProtocol", - "SemanticTypeHasherProtocol", - "BasicFileHasher", - "CachedFileHasher", - "hash_file", - "get_default_arrow_hasher", - "HashableMixin", - "hash_to_hex", - "hash_to_int", - "hash_to_uuid", - "hash_function", - "get_function_signature", - "function_content_hash", - "hash_pathset", - "hash_data", -] -``` - -- [ ] **Step 3: Update `hashing/defaults.py`** - -```python -# Default hasher accessors for the OrcaPod hashing system. - -from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeSemanticHasherRegistry -from orcapod.protocols import hashing_protocols as hp - - -def get_default_python_type_semantic_hasher_registry() -> PythonTypeSemanticHasherRegistry: - """Return the PythonTypeSemanticHasherRegistry from the default data context's semantic hasher. - - Returns: - PythonTypeSemanticHasherRegistry: The registry from the default data context. - """ - from orcapod.contexts import get_default_context - return get_default_context().semantic_hasher.type_semantic_hasher_registry - - -def get_default_semantic_hasher() -> hp.SemanticHasherProtocol: - """Return the SemanticHasherProtocol from the default data context.""" - from orcapod.contexts import get_default_context - return get_default_context().semantic_hasher - - -def get_default_arrow_hasher() -> hp.ArrowHasherProtocol: - """Return the ArrowHasherProtocol from the default data context. - - Note: file-hash caching (formerly via ``set_cacher``) has been removed. - ``StarfixArrowHasher`` does not support per-path caching. Use - ``CachedFileHasher`` when constructing a custom context if caching is needed. - """ - from orcapod.contexts import get_default_context - return get_default_context().arrow_hasher -``` - -- [ ] **Step 4: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/semantic_hashing/__init__.py \ - src/orcapod/hashing/__init__.py \ - src/orcapod/hashing/defaults.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor(hashing): update __init__.py exports and defaults for rename" -``` - ---- - -## Task 7: Update `test_semantic_hasher.py` → run tests - -**Files:** -- Modify: `tests/test_hashing/test_semantic_hasher.py` - -- [ ] **Step 1: Update imports at the top of the file** - -```python -# Old: -from orcapod.hashing.semantic_hashing.builtin_handlers import register_builtin_handlers -from orcapod.hashing.semantic_hashing.semantic_hasher import ( - BaseSemanticHasher, - _is_namedtuple, -) -from orcapod.hashing.semantic_hashing.type_handler_registry import ( - TypeHandlerRegistry, - get_default_type_handler_registry, -) - -# New: -from orcapod.hashing.semantic_hashing.builtin_handlers import ( - register_builtin_python_type_semantic_hashers, -) -from orcapod.hashing.semantic_hashing.semantic_hasher import ( - SemanticAwarePythonHasher, - _is_namedtuple, -) -from orcapod.hashing.semantic_hashing.type_handler_registry import ( - PythonTypeSemanticHasherRegistry, - get_default_python_type_semantic_hasher_registry, -) -``` - -- [ ] **Step 2: Update `make_hasher()` fixture and type annotations** - -```python -def make_hasher(strict: bool = True) -> SemanticAwarePythonHasher: - """Create a fresh SemanticAwarePythonHasher with an isolated registry.""" - registry = PythonTypeSemanticHasherRegistry() - register_builtin_python_type_semantic_hashers(registry) - return SemanticAwarePythonHasher( - hasher_id="test_v1", type_semantic_hasher_registry=registry, strict=strict - ) - - -@pytest.fixture -def hasher() -> SemanticAwarePythonHasher: - return make_hasher(strict=True) - - -@pytest.fixture -def lenient_hasher() -> SemanticAwarePythonHasher: - return make_hasher(strict=False) -``` - -- [ ] **Step 3: Update `_DummyHandler` in `TestTypeHandlerRegistry` (near line 827)** - -```python -# Old: -class _DummyHandler: - def __init__(self, tag: str) -> None: - self.tag = tag - - def handle(self, obj: Any, hasher: Any) -> Any: - return f"{self.tag}:{obj}" - -# New: -class _DummySemanticHasher: - def __init__(self, tag: str) -> None: - self.tag = tag - - def hash(self, obj: Any, hasher: Any) -> Any: - # Returns a ContentHash by delegating to the outer hasher - return hasher.hash_object(f"{self.tag}:{obj}") -``` - -- [ ] **Step 4: Update `TestTypeHandlerRegistry` class — rename class, method calls, and dummy handler** - -Rename the test class to `TestPythonTypeSemanticHasherRegistry` and update every reference: -- `TypeHandlerRegistry()` → `PythonTypeSemanticHasherRegistry()` -- `_DummyHandler(...)` → `_DummySemanticHasher(...)` -- `reg.get_handler(...)` → `reg.get_semantic_hasher(...)` -- `reg.has_handler(...)` → `reg.has_semantic_hasher(...)` -- `reg.get_handler_for_type(...)` → `reg.get_semantic_hasher_for_type(...)` - -Example of updated test methods: -```python -class TestPythonTypeSemanticHasherRegistry: - def test_register_and_get_exact(self): - reg = PythonTypeSemanticHasherRegistry() - h = _DummySemanticHasher("base") - reg.register(Base, h) - assert reg.get_semantic_hasher(Base()) is h - - def test_mro_lookup_child(self): - reg = PythonTypeSemanticHasherRegistry() - h = _DummySemanticHasher("base") - reg.register(Base, h) - assert reg.get_semantic_hasher(Child()) is h - - def test_mro_lookup_grandchild(self): - reg = PythonTypeSemanticHasherRegistry() - h = _DummySemanticHasher("base") - reg.register(Base, h) - assert reg.get_semantic_hasher(GrandChild()) is h - - def test_more_specific_handler_wins(self): - reg = PythonTypeSemanticHasherRegistry() - h_base = _DummySemanticHasher("base") - h_child = _DummySemanticHasher("child") - reg.register(Base, h_base) - reg.register(Child, h_child) - assert reg.get_semantic_hasher(Child()) is h_child - assert reg.get_semantic_hasher(GrandChild()) is h_child - - def test_unregistered_returns_none(self): - reg = PythonTypeSemanticHasherRegistry() - assert reg.get_semantic_hasher(Base()) is None - - def test_unregister_removes_handler(self): - reg = PythonTypeSemanticHasherRegistry() - h = _DummySemanticHasher("base") - reg.register(Base, h) - assert reg.unregister(Base) is True - assert reg.get_semantic_hasher(Base()) is None - - def test_unregister_nonexistent_returns_false(self): - reg = PythonTypeSemanticHasherRegistry() - assert reg.unregister(Base) is False - - def test_replace_existing_handler(self): - reg = PythonTypeSemanticHasherRegistry() - h1 = _DummySemanticHasher("first") - h2 = _DummySemanticHasher("second") - reg.register(Base, h1) - reg.register(Base, h2) - assert reg.get_semantic_hasher(Base()) is h2 - - def test_register_non_type_raises(self): - reg = PythonTypeSemanticHasherRegistry() - with pytest.raises(TypeError): - reg.register("not_a_type", _DummySemanticHasher("x")) # type: ignore[arg-type] - - def test_has_semantic_hasher_exact(self): - reg = PythonTypeSemanticHasherRegistry() - reg.register(Base, _DummySemanticHasher("b")) - assert reg.has_semantic_hasher(Base) is True - - def test_has_semantic_hasher_via_mro(self): - reg = PythonTypeSemanticHasherRegistry() - reg.register(Base, _DummySemanticHasher("b")) - assert reg.has_semantic_hasher(Child) is True - - def test_has_semantic_hasher_false(self): - reg = PythonTypeSemanticHasherRegistry() - assert reg.has_semantic_hasher(Base) is False - - def test_registered_types_snapshot(self): - reg = PythonTypeSemanticHasherRegistry() - reg.register(Base, _DummySemanticHasher("b")) - reg.register(Child, _DummySemanticHasher("c")) - types = reg.registered_types() - assert Base in types - assert Child in types - - def test_len(self): - reg = PythonTypeSemanticHasherRegistry() - assert len(reg) == 0 - reg.register(Base, _DummySemanticHasher("b")) - assert len(reg) == 1 - reg.register(Child, _DummySemanticHasher("c")) - assert len(reg) == 2 - - def test_get_semantic_hasher_for_type(self): - reg = PythonTypeSemanticHasherRegistry() - h = _DummySemanticHasher("b") - reg.register(Base, h) - assert reg.get_semantic_hasher_for_type(Base) is h - assert reg.get_semantic_hasher_for_type(Child) is h # via MRO - assert reg.get_semantic_hasher_for_type(int) is None -``` - -Also update any remaining references in the file body to `get_default_type_handler_registry` → `get_default_python_type_semantic_hasher_registry`, and any fixture type annotations. - -- [ ] **Step 5: Run tests** - -```bash -uv run --project /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - pytest tests/test_hashing/test_semantic_hasher.py -x -v -``` - -Expected: all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add tests/test_hashing/test_semantic_hasher.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "test(semantic_hasher): update for registry rename and hash() protocol tightening" -``` - ---- - -## Task 8: Add `visit_extension` to `ArrowTypeDataVisitor` + rewrite `SemanticHashingVisitor` - -**Files:** -- Modify: `src/orcapod/hashing/visitors.py` - -- [ ] **Step 1: Write a failing test for `visit_extension` dispatch** - -Create `tests/test_hashing/test_extension_type_hashing.py`: - -```python -"""Tests for extension type column hashing via SemanticHashingVisitor.""" - -from __future__ import annotations - -import pyarrow as pa -import pytest -from pathlib import Path - -from orcapod.hashing.visitors import SemanticHashingVisitor -from orcapod.contexts import get_default_context - - -@pytest.fixture -def ctx(): - return get_default_context() - - -class TestArrowTypeDataVisitorExtension: - def test_visit_dispatches_to_visit_extension_for_extension_types(self, ctx): - """visit() routes ExtensionType columns to visit_extension(), not visit_struct().""" - arrow_type = ctx.type_converter.register_python_class(Path) - assert isinstance(arrow_type, pa.ExtensionType), ( - "Path must be registered as an Arrow extension type" - ) - - calls = [] - - class TrackingVisitor(SemanticHashingVisitor): - def visit_extension(self, ext_type, storage_value): - calls.append("visit_extension") - return super().visit_extension(ext_type, storage_value) - - def visit_struct(self, struct_type, data): - calls.append("visit_struct") - return super().visit_struct(struct_type, data) - - visitor = TrackingVisitor(ctx.type_converter, ctx.semantic_hasher) - # Any value is fine for this dispatch test — use a dummy string (storage for Path is str) - visitor.visit(arrow_type, "/tmp/dummy") - assert "visit_extension" in calls - assert "visit_struct" not in calls - - -class TestSemanticHashingVisitorExtension: - def test_path_column_hashed_to_large_binary(self, ctx, tmp_path): - """Path extension columns are replaced with pa.large_binary() hash tokens.""" - file = tmp_path / "test.txt" - file.write_text("hello") - - arrow_type = ctx.type_converter.register_python_class(Path) - storage_val = ctx.type_converter.python_to_storage(Path(file), Path) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(arrow_type, storage_val) - - assert new_type == pa.large_binary() - assert isinstance(new_data, bytes) - - def test_same_content_same_hash(self, ctx, tmp_path): - """Two paths pointing to files with identical content produce the same hash bytes.""" - file1 = tmp_path / "a.txt" - file2 = tmp_path / "b.txt" - file1.write_text("identical content") - file2.write_text("identical content") - - arrow_type = ctx.type_converter.register_python_class(Path) - storage1 = ctx.type_converter.python_to_storage(Path(file1), Path) - storage2 = ctx.type_converter.python_to_storage(Path(file2), Path) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - _, hash1 = visitor.visit(arrow_type, storage1) - _, hash2 = visitor.visit(arrow_type, storage2) - - assert hash1 == hash2 - - def test_different_content_different_hash(self, ctx, tmp_path): - """Files with different content produce different hash bytes.""" - file1 = tmp_path / "x.txt" - file2 = tmp_path / "y.txt" - file1.write_text("content A") - file2.write_text("content B") - - arrow_type = ctx.type_converter.register_python_class(Path) - storage1 = ctx.type_converter.python_to_storage(Path(file1), Path) - storage2 = ctx.type_converter.python_to_storage(Path(file2), Path) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - _, hash1 = visitor.visit(arrow_type, storage1) - _, hash2 = visitor.visit(arrow_type, storage2) - - assert hash1 != hash2 - - def test_binary_encoding_format(self, ctx, tmp_path): - """Hash bytes have format b':::'.""" - file = tmp_path / "test.txt" - file.write_text("test") - - arrow_type = ctx.type_converter.register_python_class(Path) - storage_val = ctx.type_converter.python_to_storage(Path(file), Path) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - _, hash_bytes = visitor.visit(arrow_type, storage_val) - - assert b"::" in hash_bytes - type_prefix, hash_part = hash_bytes.split(b"::", 1) - # Extension name "orcapod.path" → dots replaced with colons - assert type_prefix == b"orcapod:path" - # hash_part should be "method:digest" — at least one colon - assert b":" in hash_part - - def test_null_value_passthrough(self, ctx): - """Null storage values pass through as-is.""" - arrow_type = ctx.type_converter.register_python_class(Path) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(arrow_type, None) - - assert new_type == arrow_type - assert new_data is None -``` - -- [ ] **Step 2: Run tests — verify they fail** - -```bash -uv run --project /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - pytest tests/test_hashing/test_extension_type_hashing.py -x -v -``` - -Expected: ImportError or AttributeError (methods don't exist yet). - -- [ ] **Step 3: Rewrite `visitors.py`** - -```python -""" -Generic visitor pattern for traversing Arrow types and data simultaneously. -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any - -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher -else: - pa = LazyModule("pyarrow") - - -class ArrowTypeDataVisitor(ABC): - """Base visitor for traversing Arrow types and data simultaneously.""" - - @abstractmethod - def visit_struct( - self, struct_type: "pa.StructType", data: dict | None - ) -> tuple["pa.DataType", Any]: - """Visit a struct type with its data.""" - pass - - @abstractmethod - def visit_list( - self, list_type: "pa.ListType", data: list | None - ) -> tuple["pa.DataType", Any]: - """Visit a list type with its data.""" - pass - - @abstractmethod - def visit_map( - self, map_type: "pa.MapType", data: dict | None - ) -> tuple["pa.DataType", Any]: - """Visit a map type with its data.""" - pass - - @abstractmethod - def visit_primitive( - self, primitive_type: "pa.DataType", data: Any - ) -> tuple["pa.DataType", Any]: - """Visit a primitive type with its data.""" - pass - - def visit_extension( - self, - extension_type: "pa.ExtensionType", - storage_value: Any, - ) -> tuple["pa.DataType", Any]: - """Handle an Arrow extension type. - - Default implementation: passthrough — preserves the extension type and its - storage value unchanged so that the downstream ``StarfixArrowHasher`` / - ``ArrowDigester`` sees the full extension metadata when it receives the - pre-processed table. - - Subclasses may override to convert recognised extension types to a hashed - ``pa.large_binary()`` value. - - Args: - extension_type: The Arrow extension type. - storage_value: The storage-level value (result of ``to_pylist()`` on the column). - - Returns: - Tuple of ``(new_arrow_type, new_data)``. - """ - return extension_type, storage_value - - def visit(self, arrow_type: "pa.DataType", data: Any) -> tuple["pa.DataType", Any]: - """Main dispatch method that routes to the appropriate visit method. - - Extension types are checked **first** — before the struct check — because - extension types with struct storage would otherwise be incorrectly routed - into ``visit_struct``. After ``visit_extension``, the result is re-visited - only if the type changed AND is no longer an extension type (enables - composability, avoids infinite recursion). - - Args: - arrow_type: Arrow data type to process. - data: Corresponding data value. - - Returns: - Tuple of ``(new_arrow_type, new_data)``. - """ - if isinstance(arrow_type, pa.ExtensionType): - new_type, new_data = self.visit_extension(arrow_type, data) - if new_type is not arrow_type and not isinstance(new_type, pa.ExtensionType): - return self.visit(new_type, new_data) - return new_type, new_data - - if pa.types.is_struct(arrow_type): - return self.visit_struct(arrow_type, data) - elif pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type): - return self.visit_list(arrow_type, data) - elif pa.types.is_fixed_size_list(arrow_type): - return self.visit_list(arrow_type, data) - elif pa.types.is_map(arrow_type): - return self.visit_map(arrow_type, data) - else: - return self.visit_primitive(arrow_type, data) - - def _visit_struct_fields( - self, struct_type: "pa.StructType", data: dict | None - ) -> tuple["pa.StructType", dict]: - """Recursively process struct fields. Default behavior for regular structs.""" - if data is None: - return struct_type, None - - new_fields = [] - new_data = {} - - for field in struct_type: - field_data = data.get(field.name) - new_field_type, new_field_data = self.visit(field.type, field_data) - new_fields.append(pa.field(field.name, new_field_type)) - new_data[field.name] = new_field_data - - return pa.struct(new_fields), new_data - - def _visit_list_elements( - self, list_type: "pa.ListType", data: list | None - ) -> tuple["pa.DataType", list]: - """Recursively process list elements.""" - if data is None: - return list_type, None - - element_type = list_type.value_type - processed_elements = [] - new_element_type = None - - for item in data: - current_element_type, processed_item = self.visit(element_type, item) - processed_elements.append(processed_item) - if new_element_type is None: - new_element_type = current_element_type - - if new_element_type is None: - new_element_type = element_type - - if pa.types.is_large_list(list_type): - return pa.large_list(new_element_type), processed_elements - elif pa.types.is_fixed_size_list(list_type): - return pa.list_(new_element_type, list_type.list_size), processed_elements - else: - return pa.list_(new_element_type), processed_elements - - -class SemanticHashingError(Exception): - """Exception raised when semantic hashing fails.""" - pass - - -class SemanticHashingVisitor(ArrowTypeDataVisitor): - """Visitor that replaces extension-typed columns with their content hashes. - - For each Arrow column whose type is a ``pa.ExtensionType``: - - 1. Look up the corresponding Python type via ``type_converter``. - 2. If the Python type has a semantic hasher registered in ``python_hasher``, - convert the storage value to a Python object and hash it, replacing the - column with a ``pa.large_binary()`` value of the form:: - - + b"::" + content_hash.to_prefixed_digest() - - where ``type_name`` is the extension name with dots replaced by colons - (e.g. ``"orcapod.path"`` → ``"orcapod:path"``), and - ``to_prefixed_digest()`` = ``method_bytes + b":" + digest``. - 3. If no hasher is registered (or the converter doesn't know the type), - return the extension type and storage value unchanged. The downstream - ``StarfixArrowHasher`` / ``ArrowDigester`` will see the full extension - metadata intact and hash it in a type-aware way. - - Args: - type_converter: The active ``UniversalTypeConverter`` for resolving - extension type → Python type and storage → Python conversion. - python_hasher: The active ``SemanticAwarePythonHasher`` for hashing - Python objects. - """ - - def __init__( - self, - type_converter: "UniversalTypeConverter", - python_hasher: "SemanticAwarePythonHasher", - ) -> None: - self._type_converter = type_converter - self._python_hasher = python_hasher - self._current_field_path: list[str] = [] - - def visit_extension( - self, - extension_type: "pa.ExtensionType", - storage_value: Any, - ) -> tuple["pa.DataType", Any]: - """Hash an extension type value to pa.large_binary(), or passthrough.""" - if storage_value is None: - return extension_type, None - - from typing import Any as _Any - - # Resolve extension type → Python type. - python_type = self._type_converter.arrow_type_to_python_type(extension_type) - - # If the converter couldn't resolve to a concrete class, passthrough. - if python_type is _Any or not isinstance(python_type, type): - return extension_type, storage_value - - # Only hash if a semantic hasher is registered for this Python type. - if not self._python_hasher.type_semantic_hasher_registry.has_semantic_hasher( - python_type - ): - return extension_type, storage_value - - # Convert storage value → Python object and hash it. - python_obj = self._type_converter.storage_to_python(storage_value, python_type) - content_hash = self._python_hasher.hash_object(python_obj) - - # Encode as binary: ":::" - # Dots in the extension name → colons (e.g. "orcapod.path" → "orcapod:path"). - # The "::" separator is unambiguous because to_prefixed_digest() uses only ":". - type_name = extension_type.extension_name.replace(".", ":") - hash_bytes = ( - type_name.encode("ascii") - + b"::" - + content_hash.to_prefixed_digest() - ) - return pa.large_binary(), hash_bytes - - def visit_struct( - self, struct_type: "pa.StructType", data: dict | None - ) -> tuple["pa.DataType", Any]: - """Regular struct (no extension identity) — recurse into fields.""" - if data is None: - return struct_type, None - return self._visit_struct_fields(struct_type, data) - - def visit_list( - self, list_type: "pa.ListType", data: list | None - ) -> tuple["pa.DataType", Any]: - """Recurse into list elements.""" - if data is None: - return list_type, None - self._current_field_path.append("[*]") - try: - return self._visit_list_elements(list_type, data) - finally: - self._current_field_path.pop() - - def visit_map( - self, map_type: "pa.MapType", data: dict | None - ) -> tuple["pa.DataType", Any]: - """Pass map types through unchanged.""" - return map_type, data - - def visit_primitive( - self, primitive_type: "pa.DataType", data: Any - ) -> tuple["pa.DataType", Any]: - """Pass primitive types through unchanged.""" - return primitive_type, data - - def _visit_struct_fields( - self, struct_type: "pa.StructType", data: dict | None - ) -> tuple["pa.StructType", dict]: - """Override to add field path tracking for better error messages.""" - if data is None: - return struct_type, None - - new_fields = [] - new_data = {} - - for field in struct_type: - self._current_field_path.append(field.name) - try: - field_data = data.get(field.name) - new_field_type, new_field_data = self.visit(field.type, field_data) - new_fields.append(pa.field(field.name, new_field_type)) - new_data[field.name] = new_field_data - finally: - self._current_field_path.pop() - - return pa.struct(new_fields), new_data -``` - -- [ ] **Step 4: Run tests — verify they pass** - -```bash -uv run --project /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - pytest tests/test_hashing/test_extension_type_hashing.py -x -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/visitors.py \ - tests/test_hashing/test_extension_type_hashing.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "feat(visitors): add visit_extension dispatch; rewrite SemanticHashingVisitor for extension types" -``` - ---- - -## Task 9: Update `StarfixArrowHasher`, delete `SemanticArrowHasher` - -**Files:** -- Modify: `src/orcapod/hashing/arrow_hashers.py` - -- [ ] **Step 1: Rewrite `arrow_hashers.py`** - -Delete the entire `SemanticArrowHasher` class. Update `StarfixArrowHasher`: - -```python -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -import pyarrow as pa -from starfix import ArrowDigester - -from orcapod.hashing.schema_cleaner import clean_schema_for_hashing, has_extension_metadata -from orcapod.hashing.visitors import SemanticHashingVisitor -from orcapod.types import ContentHash -from orcapod.utils import arrow_utils - -if TYPE_CHECKING: - from orcapod.semantic_types.universal_converter import UniversalTypeConverter - from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher - - -class StarfixArrowHasher: - """Arrow table hasher backed by the starfix-python ``ArrowDigester``. - - Pipeline - -------- - 1. **Semantic pre-processing** — the ``SemanticHashingVisitor`` traverses - every column. Extension-typed columns whose Python type has a registered - semantic hasher are replaced with ``pa.large_binary()`` hash tokens - (e.g. ``Path`` columns are replaced by their file-content hash). - Extension-typed columns without a registered hasher pass through with - their full extension metadata intact. - 2. **Starfix hashing** — ``ArrowDigester.hash_table`` produces a 35-byte - versioned SHA-256 digest that is byte-for-byte identical to the Rust - ``starfix`` crate output. - - Parameters - ---------- - type_converter: - ``UniversalTypeConverter`` used to resolve extension types to Python - types and convert storage values back to Python objects. - semantic_hasher: - ``SemanticAwarePythonHasher`` used to hash Python objects extracted - from extension-typed columns. - hasher_id: - String identifier embedded in every ``ContentHash`` produced by this - hasher. - """ - - def __init__( - self, - type_converter: "UniversalTypeConverter", - semantic_hasher: "SemanticAwarePythonHasher", - hasher_id: str, - ) -> None: - self._type_converter = type_converter - self._semantic_hasher = semantic_hasher - self._hasher_id = hasher_id - - @property - def hasher_id(self) -> str: - return self._hasher_id - - def _process_table_columns(self, table: "pa.Table | pa.RecordBatch") -> "pa.Table": - """Replace semantic-typed columns with their content-hash bytes.""" - new_columns: list[pa.Array] = [] - new_fields: list[pa.Field] = [] - - for i, field in enumerate(table.schema): - # Short-circuit: columns that cannot contain semantic types skip - # the costly Python round-trip. Extension types must pass through - # so visit_extension can process them. - if not ( - isinstance(field.type, pa.ExtensionType) - or pa.types.is_struct(field.type) - or pa.types.is_list(field.type) - or pa.types.is_large_list(field.type) - or pa.types.is_fixed_size_list(field.type) - or pa.types.is_map(field.type) - ): - new_columns.append(table.column(i)) - new_fields.append(field) - continue - - column_data = table.column(i).to_pylist() - visitor = SemanticHashingVisitor(self._type_converter, self._semantic_hasher) - - try: - new_type: pa.DataType | None = None - processed_data: list[Any] = [] - for value in column_data: - processed_type, processed_value = visitor.visit(field.type, value) - if new_type is None and processed_value is not None: - new_type = processed_type - processed_data.append(processed_value) - - if new_type is None: - new_type = field.type - new_columns.append(pa.array(processed_data, type=new_type)) - new_fields.append(field.with_type(new_type)) - - except Exception as exc: - raise RuntimeError( - f"Failed to process column '{field.name}': {exc}" - ) from exc - - return pa.table( - new_columns, - schema=pa.schema(new_fields, metadata=table.schema.metadata), - ) - - def hash_schema(self, schema: "pa.Schema") -> ContentHash: - """Hash an Arrow schema using the starfix canonical algorithm.""" - include_meta = has_extension_metadata(schema) - if include_meta: - schema = clean_schema_for_hashing(schema) - digest = ArrowDigester.hash_schema(schema, include_metadata=include_meta) - return ContentHash(method=self._hasher_id, digest=digest) - - def hash_table(self, table: "pa.Table | pa.RecordBatch") -> ContentHash: - """Hash an Arrow table (or ``RecordBatch``) using starfix.""" - if isinstance(table, pa.RecordBatch): - table = pa.Table.from_batches([table]) - - processed_table = self._process_table_columns(table) - include_meta = has_extension_metadata(processed_table.schema) - if include_meta: - clean_schema = clean_schema_for_hashing(processed_table.schema) - clean_table = pa.Table.from_arrays( - processed_table.columns, schema=clean_schema - ) - else: - clean_table = processed_table - digest = ArrowDigester.hash_table(clean_table, include_metadata=include_meta) - return ContentHash(method=self._hasher_id, digest=digest) -``` - -- [ ] **Step 2: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/hashing/arrow_hashers.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "refactor(arrow_hashers): update StarfixArrowHasher for extension types, delete SemanticArrowHasher" -``` - ---- - -## Task 10: Update `test_starfix_arrow_hasher.py`, run tests - -**Files:** -- Modify: `tests/test_hashing/test_starfix_arrow_hasher.py` - -- [ ] **Step 1: Update `_make_hasher()` and remove `SemanticTypeRegistry` import** - -```python -# Remove this import: -# from orcapod.semantic_types import SemanticTypeRegistry - -# Update _make_hasher(): -def _make_hasher() -> StarfixArrowHasher: - from orcapod.contexts import get_default_context - ctx = get_default_context() - return StarfixArrowHasher( - type_converter=ctx.type_converter, - semantic_hasher=ctx.semantic_hasher, - hasher_id=HASHER_ID, - ) -``` - -- [ ] **Step 2: Run the hashing test suite** - -```bash -uv run --project /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - pytest tests/test_hashing/ -x -v -``` - -Expected: all tests pass (golden digests unchanged for plain-schema tables; extension type tests pass). - -- [ ] **Step 3: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add tests/test_hashing/test_starfix_arrow_hasher.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "test(starfix_arrow_hasher): update _make_hasher() for new constructor, remove SemanticTypeRegistry import" -``` - ---- - -## Task 11: Update `v0.1.json`, `context_schema.json`, and `versioned_hashers.py` - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` -- Modify: `src/orcapod/contexts/data/schemas/context_schema.json` -- Modify: `src/orcapod/hashing/versioned_hashers.py` - -- [ ] **Step 1: Rewrite `v0.1.json`** - -Key design note: `arrow_hasher` now depends on `semantic_hasher`, and `semantic_hasher` depends on `python_type_semantic_hasher_registry`. To avoid a circular dependency, the `pa.Table`/`pa.RecordBatch` handler entries are **removed** from the registry's handlers list (those entries previously referenced `arrow_hasher`). The JSON construction order is: `file_hasher` → `type_converter` → `function_info_extractor` → `python_type_semantic_hasher_registry` → `semantic_hasher` → `arrow_hasher`. - -```json -{ - "context_key": "std:v0.1:default", - "version": "v0.1", - "description": "Initial stable release with extension type hashing support", - "file_hasher": { - "_class": "orcapod.hashing.file_hashers.BasicFileHasher", - "_config": { - "algorithm": "sha256" - } - }, - "type_converter": { - "_class": "orcapod.semantic_types.universal_converter.UniversalTypeConverter", - "_config": { - "logical_type_registry": { - "_class": "orcapod.extension_types.registry.LogicalTypeRegistry", - "_config": { - "logical_types": [ - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUPath", - "_config": {} - }, - { - "_class": "orcapod.extension_types.builtin_logical_types.LogicalUUID", - "_config": {} - } - ], - "factories": [ - { - "factory": { - "_class": "orcapod.extension_types.dataclass_logical_type_factory.DataclassLogicalTypeFactory", - "_config": {} - }, - "category": "orcapod.dataclass", - "python_bases": [{"_type": "builtins.object"}] - }, - { - "factory": { - "_class": "orcapod.extension_types.pydantic_logical_type_factory.PydanticLogicalTypeFactory", - "_config": {} - }, - "category": "orcapod.pydantic", - "python_bases": [{"_type": "pydantic.BaseModel"}] - } - ] - } - } - } - }, - "function_info_extractor": { - "_class": "orcapod.hashing.semantic_hashing.function_info_extractors.FunctionSignatureExtractor", - "_config": { - "include_module": true, - "include_defaults": true - } - }, - "python_type_semantic_hasher_registry": { - "_class": "orcapod.hashing.semantic_hashing.type_handler_registry.PythonTypeSemanticHasherRegistry", - "_config": { - "handlers": [ - [{"_type": "builtins.bytes"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.BytesSemanticHasher", "_config": {}}], - [{"_type": "builtins.bytearray"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.BytesSemanticHasher", "_config": {}}], - [{"_type": "pathlib.Path"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PathSemanticHasher", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], - [{"_type": "upath.core.UPath"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UPathSemanticHasher", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], - [{"_type": "uuid.UUID"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UUIDSemanticHasher", "_config": {}}], - [{"_type": "types.FunctionType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FunctionSemanticHasher", "_config": {"function_info_extractor": {"_ref": "function_info_extractor"}}}], - [{"_type": "types.BuiltinFunctionType"},{"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FunctionSemanticHasher", "_config": {"function_info_extractor": {"_ref": "function_info_extractor"}}}], - [{"_type": "types.MethodType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FunctionSemanticHasher", "_config": {"function_info_extractor": {"_ref": "function_info_extractor"}}}], - [{"_type": "builtins.type"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.TypeObjectSemanticHasher", "_config": {}}], - [{"_type": "types.GenericAlias"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.GenericAliasSemanticHasher", "_config": {}}], - [{"_type": "types.UnionType"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UnionTypeSemanticHasher", "_config": {}}], - [{"_type": "typing._GenericAlias"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.GenericAliasSemanticHasher", "_config": {}}], - [{"_type": "typing._SpecialForm"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.SpecialFormSemanticHasher", "_config": {}}] - ] - } - }, - "semantic_hasher": { - "_class": "orcapod.hashing.semantic_hashing.semantic_hasher.SemanticAwarePythonHasher", - "_config": { - "hasher_id": "semantic_v0.1", - "type_semantic_hasher_registry": { - "_ref": "python_type_semantic_hasher_registry" - } - } - }, - "arrow_hasher": { - "_class": "orcapod.hashing.arrow_hashers.StarfixArrowHasher", - "_config": { - "hasher_id": "arrow_v0.1", - "type_converter": {"_ref": "type_converter"}, - "semantic_hasher": {"_ref": "semantic_hasher"} - } - }, - "metadata": { - "created_date": "2026-06-24", - "author": "OrcaPod Core Team", - "changelog": [ - "Initial release with Path semantic type support", - "Basic SHA-256 hashing for files and objects", - "Arrow logical serialization method", - "Introduced arrow_v0.1 StarfixArrowHasher using starfix ArrowDigester for cross-language-compatible Arrow hashing", - "Hard cut: replaced shape-based SemanticTypeRegistry with extension-type hashing; renamed all hashing classes to clearer names" - ] - } -} -``` - -- [ ] **Step 2: Update `context_schema.json`** - -Two changes: -1. Remove the `semantic_registry` property from `properties`. -2. Rename `type_handler_registry` → `python_type_semantic_hasher_registry` in `properties`. - -```json -"python_type_semantic_hasher_registry": { - "$ref": "#/$defs/objectspec", - "description": "ObjectSpec for the PythonTypeSemanticHasherRegistry used by the semantic hasher" -}, -``` - -Also update the `examples` section references and remove the `"semantic_registry"` entry. - -- [ ] **Step 3: Update `versioned_hashers.py`** - -```python -""" -Versioned hasher factories for OrcaPod. -""" - -from __future__ import annotations - -import logging -from typing import Any - -from orcapod.protocols import hashing_protocols as hp - -logger = logging.getLogger(__name__) - -_CURRENT_SEMANTIC_HASHER_ID = "semantic_v0.1" -_CURRENT_ARROW_HASHER_ID = "arrow_v0.1" - - -def get_versioned_semantic_hasher( - hasher_id: str = _CURRENT_SEMANTIC_HASHER_ID, - strict: bool = True, - type_semantic_hasher_registry: "Any | None" = None, -) -> hp.SemanticHasherProtocol: - """Return a SemanticAwarePythonHasher configured for the current version. - - Parameters - ---------- - hasher_id: - Identifier embedded in every ContentHash produced by this hasher. - strict: - When True raises TypeError for unhandled types. When False falls back - to a best-effort string representation. - type_semantic_hasher_registry: - Optional ``PythonTypeSemanticHasherRegistry`` to inject. When None the - global default registry is used. - """ - from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher - - if type_semantic_hasher_registry is None: - from orcapod.hashing.semantic_hashing.type_handler_registry import ( - get_default_python_type_semantic_hasher_registry, - ) - type_semantic_hasher_registry = get_default_python_type_semantic_hasher_registry() - - logger.debug( - "get_versioned_semantic_hasher: creating SemanticAwarePythonHasher " - "(hasher_id=%r, strict=%r)", - hasher_id, - strict, - ) - return SemanticAwarePythonHasher( - hasher_id=hasher_id, - type_semantic_hasher_registry=type_semantic_hasher_registry, - strict=strict, - ) - - -def get_versioned_semantic_arrow_hasher( - hasher_id: str = _CURRENT_ARROW_HASHER_ID, -) -> hp.ArrowHasherProtocol: - """Return a StarfixArrowHasher configured for the current version. - - Sources ``type_converter`` and ``semantic_hasher`` from the default - ``DataContext`` so that the arrow hasher is consistent with all other - versioned components. - - Parameters - ---------- - hasher_id: - Identifier embedded in every ContentHash produced by this hasher. - """ - from orcapod.hashing.arrow_hashers import StarfixArrowHasher - from orcapod.contexts import resolve_context - - ctx = resolve_context(None) # default context - logger.debug( - "get_versioned_semantic_arrow_hasher: creating StarfixArrowHasher " - "(hasher_id=%r)", - hasher_id, - ) - return StarfixArrowHasher( - hasher_id=hasher_id, - type_converter=ctx.type_converter, - semantic_hasher=ctx.semantic_hasher, - ) -``` - -- [ ] **Step 4: Run the full test suite (except test_semantic_types)** - -```bash -uv run --project /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - pytest tests/test_hashing/ tests/test_extension_types/ tests/test_core/ -x -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add src/orcapod/contexts/data/v0.1.json \ - src/orcapod/contexts/data/schemas/context_schema.json \ - src/orcapod/hashing/versioned_hashers.py -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "feat(v0.1): wire extension type hashing into default context; remove semantic_registry" -``` - ---- - -## Task 12: Delete old semantic type system + grep sweep + final test run - -**Files:** -- Delete: `src/orcapod/semantic_types/semantic_struct_converters.py` -- Delete: `src/orcapod/semantic_types/semantic_registry.py` -- Delete: `tests/test_semantic_types/` (all 9 files) -- Delete: `tests/test_hashing/test_file_hashing_consistency.py` -- Modify: `src/orcapod/semantic_types/__init__.py` -- Modify: `src/orcapod/protocols/semantic_types_protocols.py` - -- [ ] **Step 1: Delete old source files** - -```bash -rm /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python/src/orcapod/semantic_types/semantic_struct_converters.py -rm /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python/src/orcapod/semantic_types/semantic_registry.py -``` - -- [ ] **Step 2: Update `semantic_types/__init__.py`** — remove `SemanticTypeRegistry` export - -```python -from .universal_converter import UniversalTypeConverter -from .type_inference import infer_python_schema_from_pylist_data - -__all__ = [ - "UniversalTypeConverter", - "infer_python_schema_from_pylist_data", -] -``` - -- [ ] **Step 3: Remove `SemanticStructConverterProtocol` from `semantic_types_protocols.py`** - -Delete the `SemanticStructConverterProtocol` class and any imports that only support it. Keep `TypeConverterProtocol` and all other classes. - -- [ ] **Step 4: Delete old test files** - -```bash -rm /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python/tests/test_hashing/test_file_hashing_consistency.py -rm -r /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python/tests/test_semantic_types/ -``` - -- [ ] **Step 5: Grep sweep for stale references** - -```bash -grep -rn \ - "SemanticTypeRegistry\|semantic_registry\|SemanticStructConverter\ -\|BaseSemanticHasher\|TypeHandlerRegistry\|BuiltinTypeHandlerRegistry\ -\|TypeHandlerProtocol\|PathContentHandler\|UPathContentHandler\ -\|UUIDHandler\|BytesHandler\|FunctionHandler\|TypeObjectHandler\ -\|SpecialFormHandler\|GenericAliasHandler\|UnionTypeHandler\|ArrowTableHandler\ -\|SchemaHandler\|register_builtin_handlers\|get_default_type_handler_registry\ -\|type_handler_registry\|get_handler\b\|has_handler\b\|SemanticArrowHasher" \ - /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python/src/ \ - /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python/tests/ \ - 2>/dev/null -``` - -Expected: zero matches (fix any that appear before continuing). - -- [ ] **Step 6: Run full test suite** - -```bash -uv run --project /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - pytest tests/test_hashing/ tests/test_extension_types/ tests/test_core/ -x -v -``` - -Expected: all tests pass. - -- [ ] **Step 7: Final commit** - -```bash -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - add -u -git -C /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python \ - commit -m "feat(PLT-1660): hard cut — delete SemanticTypeRegistry and old struct-based hashing system" -``` - ---- - -## Self-Review - -**Spec coverage:** -- ✅ §1 `visit_extension` added to `ArrowTypeDataVisitor`, `visit()` updated (Task 8) -- ✅ §2 `SemanticHashingVisitor` rewritten with binary encoding (Task 8) -- ✅ §3 `StarfixArrowHasher` constructor updated + short-circuit + `SemanticArrowHasher` deleted (Task 9) -- ✅ §4 `SemanticArrowHasher` deleted (Task 9) -- ✅ §5 All class/method renames applied (Tasks 1–6) -- ✅ §6 Protocol tightened: `hash() -> ContentHash` (Tasks 1, 3, 4) -- ✅ §7 `v0.1.json` updated (Task 11) — note: `pa.Table`/`pa.RecordBatch` handlers removed to break circular dep -- ✅ §8 `context_schema.json` updated (Task 11) -- ✅ §9 `DataContext.core` docstring updated (Task 5) -- ✅ §10 `versioned_hashers.py` sources from context (Task 11) -- ✅ Files to delete: all covered (Task 12) -- ✅ Files to update: covered across Tasks 1–11 - -**Circular dependency note (§7 deviation):** The spec says to add `"semantic_hasher": {"_ref": "semantic_hasher"}` to `arrow_hasher._config`. This is correct and implemented. However, to avoid a construction-order cycle (`arrow_hasher` → `semantic_hasher` → `registry` → `arrow_hasher` via `ArrowTableSemanticHasher`), the `pa.Table` and `pa.RecordBatch` handler entries are removed from the `python_type_semantic_hasher_registry` handlers list in `v0.1.json`. These handlers depended on `arrow_hasher` creating the cycle. The `register_builtin_python_type_semantic_hashers()` function still supports them when `arrow_hasher` is passed explicitly (e.g., for custom registry construction in tests). - -**Type consistency check:** -- `SemanticAwarePythonHasher.__init__` takes `type_semantic_hasher_registry` → `v0.1.json` uses key `type_semantic_hasher_registry` ✅ -- `SemanticHashingVisitor.__init__` takes `type_converter, python_hasher` → `_process_table_columns` passes `self._type_converter, self._semantic_hasher` ✅ -- `StarfixArrowHasher.__init__` takes `type_converter, semantic_hasher, hasher_id` → `versioned_hashers.py` passes these by keyword ✅ -- `PythonTypeSemanticHasherRegistry.get_semantic_hasher(obj)` → `SemanticAwarePythonHasher.hash_object()` calls this ✅ -- `PythonTypeSemanticHasherRegistry.has_semantic_hasher(target_type)` → `SemanticHashingVisitor.visit_extension()` calls this ✅ diff --git a/superpowers/plans/2026-06-24-rename-semantic-hasher-to-handler.md b/superpowers/plans/2026-06-24-rename-semantic-hasher-to-handler.md deleted file mode 100644 index d33489a30..000000000 --- a/superpowers/plans/2026-06-24-rename-semantic-hasher-to-handler.md +++ /dev/null @@ -1,422 +0,0 @@ -# Rename *SemanticHasher → *Handler, PythonTypeSemanticHasherRegistry → PythonTypeHandlerRegistry - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Mechanically rename all `*SemanticHasher` handler classes to `*Handler`, all `PythonTypeSemanticHasherRegistry` variants to `PythonTypeHandlerRegistry`, and the `type_semantic_hasher_registry` param/property to `type_handler_registry` — no logic changes. - -**Architecture:** Pure find-and-replace of identifiers across ~10 source files and 2 JSON configs. Every old name maps 1-to-1 to a new name. No logic, no interface changes, no backward-compat shims (greenfield project). - -**Tech Stack:** Python, JSON, uv/pytest - ---- - -## File Map - -| File | What changes | -|---|---| -| `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | 11 class names + function name + docstring/string literals | -| `src/orcapod/hashing/semantic_hashing/type_handler_registry.py` | 3 class/function names + docstrings + internal log strings | -| `src/orcapod/hashing/semantic_hashing/semantic_hasher.py` | param + property name `type_semantic_hasher_registry` → `type_handler_registry` + docstring | -| `src/orcapod/hashing/semantic_hashing/__init__.py` | imports + `__all__` | -| `src/orcapod/hashing/__init__.py` | imports + `__all__` | -| `src/orcapod/hashing/defaults.py` | function name + import + docstring | -| `src/orcapod/hashing/versioned_hashers.py` | param name + import | -| `src/orcapod/protocols/hashing_protocols.py` | property name in `SemanticHasherProtocol` + TYPE_CHECKING import | -| `src/orcapod/contexts/data/v0.1.json` | top-level key, `_class` values, `_ref` value, sub-key | -| `src/orcapod/contexts/data/schemas/context_schema.json` | property key | -| `tests/test_hashing/test_semantic_hasher.py` | imports + usage | -| `tests/test_hashing/test_uuid_handler.py` | imports + usage | -| `tests/test_hashing/test_extension_type_hashing.py` | no old names (already clean) | -| `test-objective/unit/test_hashing.py` | imports, class names, type annotations, comments | - ---- - -## Rename Reference Table - -### Handler classes (builtin_handlers.py + all callers) - -| Old | New | -|---|---| -| `PathSemanticHasher` | `PathHandler` | -| `UPathSemanticHasher` | `UPathHandler` | -| `UUIDSemanticHasher` | `UUIDHandler` | -| `BytesSemanticHasher` | `BytesHandler` | -| `FunctionSemanticHasher` | `FunctionHandler` | -| `TypeObjectSemanticHasher` | `TypeObjectHandler` | -| `SpecialFormSemanticHasher` | `SpecialFormHandler` | -| `GenericAliasSemanticHasher` | `GenericAliasHandler` | -| `UnionTypeSemanticHasher` | `UnionTypeHandler` | -| `ArrowTableSemanticHasher` | `ArrowTableHandler` | -| `SchemaSemanticHasher` | `SchemaHandler` | -| `register_builtin_python_type_semantic_hashers` | `register_builtin_python_type_handlers` | - -### Registry classes (type_handler_registry.py + all callers) - -| Old | New | -|---|---| -| `PythonTypeSemanticHasherRegistry` | `PythonTypeHandlerRegistry` | -| `BuiltinPythonTypeSemanticHasherRegistry` | `BuiltinPythonTypeHandlerRegistry` | -| `get_default_python_type_semantic_hasher_registry` | `get_default_python_type_handler_registry` | - -### Parameter/property (semantic_hasher.py + all callers) - -| Old | New | -|---|---| -| `type_semantic_hasher_registry` | `type_handler_registry` | - ---- - -## Task 1: Rename class definitions and internal strings in `builtin_handlers.py` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` - -- [ ] **Step 1: Apply all renames in builtin_handlers.py** - - Changes needed (all are identifier or string-literal renames only): - - Module docstring: update all `*SemanticHasher` names and `register_builtin_python_type_semantic_hashers` - - TYPE_CHECKING import: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - All 11 class definitions: `class PathSemanticHasher` → `class PathHandler`, etc. - - Error messages inside class bodies: e.g. `"PathSemanticHasher: path does not exist"` → `"PathHandler: path does not exist"` - - `logger.debug` strings: e.g. `"PathSemanticHasher: hashing file content"` → `"PathHandler: hashing file content"` - - Function `register_builtin_python_type_semantic_hashers` → `register_builtin_python_type_handlers` - - Docstring inside that function: update `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Final `logger.debug` string: `"register_builtin_python_type_semantic_hashers: registered %d hashers"` → `"register_builtin_python_type_handlers: registered %d hashers"` - -- [ ] **Step 2: Verify file parses correctly** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing.semantic_hashing import builtin_handlers; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 2: Rename class definitions in `type_handler_registry.py` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/type_handler_registry.py` - -- [ ] **Step 1: Apply all renames in type_handler_registry.py** - - Changes needed: - - Module docstring: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Class `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - `__repr__` method: `"PythonTypeSemanticHasherRegistry(registered=..."` → `"PythonTypeHandlerRegistry(registered=..."` - - `logger.debug` strings that mention `PythonTypeSemanticHasherRegistry` - - Function `get_default_python_type_semantic_hasher_registry` → `get_default_python_type_handler_registry` - - The function body's import: `get_default_python_type_semantic_hasher_registry as _get` → `get_default_python_type_handler_registry as _get` - - Class `BuiltinPythonTypeSemanticHasherRegistry` → `BuiltinPythonTypeHandlerRegistry` - - Docstring: `"A PythonTypeSemanticHasherRegistry pre-populated..."` → `"A PythonTypeHandlerRegistry pre-populated..."` - - `super().__init__()` call — no change needed - - Import inside `__init__`: `register_builtin_python_type_semantic_hashers` → `register_builtin_python_type_handlers` - - Call: `register_builtin_python_type_semantic_hashers(self, ...)` → `register_builtin_python_type_handlers(self, ...)` - -- [ ] **Step 2: Verify file parses correctly** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeHandlerRegistry; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 3: Rename param/property in `semantic_hasher.py` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/semantic_hasher.py` - -- [ ] **Step 1: Apply renames in semantic_hasher.py** - - Changes needed: - - Import: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Docstring parameter: `type_semantic_hasher_registry:` → `type_handler_registry:` - - Constructor param: `type_semantic_hasher_registry: PythonTypeHandlerRegistry | None = None` → `type_handler_registry: PythonTypeHandlerRegistry | None = None` - - Constructor body: `if type_semantic_hasher_registry is None:` → `if type_handler_registry is None:` - - Constructor body: `from orcapod.hashing.defaults import get_default_python_type_semantic_hasher_registry` → `get_default_python_type_handler_registry` - - Constructor body: `self._registry = get_default_python_type_semantic_hasher_registry()` → `get_default_python_type_handler_registry()` - - Constructor body: `else: self._registry = type_semantic_hasher_registry` → `else: self._registry = type_handler_registry` - - Property `type_semantic_hasher_registry` → `type_handler_registry` - - Property docstring: `"Return the ``PythonTypeSemanticHasherRegistry``..."` → `"Return the ``PythonTypeHandlerRegistry``..."` - - Property return type annotation: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Error message in `_handle_unknown`: `"via the PythonTypeSemanticHasherRegistry or"` → `"via the PythonTypeHandlerRegistry or"` - -- [ ] **Step 2: Verify file parses correctly** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 4: Update `semantic_hashing/__init__.py` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/__init__.py` - -- [ ] **Step 1: Apply renames** - - Changes needed: - - Module docstring: all `*SemanticHasher` names → `*Handler` equivalents - - Import from `builtin_handlers`: `BytesSemanticHasher` → `BytesHandler`, etc.; `register_builtin_python_type_semantic_hashers` → `register_builtin_python_type_handlers` - - Import from `type_handler_registry`: `BuiltinPythonTypeSemanticHasherRegistry` → `BuiltinPythonTypeHandlerRegistry`, `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - `__all__`: update all entries to new names - -- [ ] **Step 2: Verify** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing.semantic_hashing import PathHandler, PythonTypeHandlerRegistry, register_builtin_python_type_handlers; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 5: Update `hashing/__init__.py` - -**Files:** -- Modify: `src/orcapod/hashing/__init__.py` - -- [ ] **Step 1: Apply renames** - - Changes needed: - - Module docstring: update all old names - - Import from `defaults`: `get_default_python_type_semantic_hasher_registry` → `get_default_python_type_handler_registry` - - Import from `builtin_handlers`: `BytesSemanticHasher` → `BytesHandler`, etc.; `register_builtin_python_type_semantic_hashers` → `register_builtin_python_type_handlers` - - Import from `type_handler_registry`: `BuiltinPythonTypeSemanticHasherRegistry` → `BuiltinPythonTypeHandlerRegistry`, `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - `__all__`: update all entries to new names - -- [ ] **Step 2: Verify** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing import PythonTypeHandlerRegistry, get_default_python_type_handler_registry, BytesHandler; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 6: Update `hashing/defaults.py` - -**Files:** -- Modify: `src/orcapod/hashing/defaults.py` - -- [ ] **Step 1: Apply renames** - - Changes needed: - - Import: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Function name: `get_default_python_type_semantic_hasher_registry` → `get_default_python_type_handler_registry` - - Return type annotation: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Docstring: update class name references - - Function body: `get_default_context().semantic_hasher.type_semantic_hasher_registry` → `get_default_context().semantic_hasher.type_handler_registry` - -- [ ] **Step 2: Verify** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing.defaults import get_default_python_type_handler_registry; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 7: Update `hashing/versioned_hashers.py` - -**Files:** -- Modify: `src/orcapod/hashing/versioned_hashers.py` - -- [ ] **Step 1: Apply renames** - - Changes needed: - - Function param: `type_semantic_hasher_registry: "Any | None" = None` → `type_handler_registry: "Any | None" = None` - - Docstring param description: `type_semantic_hasher_registry:` → `type_handler_registry:` - - Import inside function: `get_default_python_type_semantic_hasher_registry` → `get_default_python_type_handler_registry` - - Variable: `type_semantic_hasher_registry = get_default_python_type_semantic_hasher_registry()` → `type_handler_registry = get_default_python_type_handler_registry()` - - `SemanticAwarePythonHasher(... type_semantic_hasher_registry=type_semantic_hasher_registry ...)` → `... type_handler_registry=type_handler_registry ...` - -- [ ] **Step 2: Verify** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.hashing.versioned_hashers import get_versioned_semantic_hasher; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 8: Update `protocols/hashing_protocols.py` - -**Files:** -- Modify: `src/orcapod/protocols/hashing_protocols.py` - -- [ ] **Step 1: Apply renames** - - Changes needed: - - TYPE_CHECKING import: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - `SemanticHasherProtocol.type_semantic_hasher_registry` property → `type_handler_registry` - - Property docstring: `"Return the PythonTypeSemanticHasherRegistry..."` → `"Return the PythonTypeHandlerRegistry..."` - - Property return type annotation: `"PythonTypeSemanticHasherRegistry"` → `"PythonTypeHandlerRegistry"` - -- [ ] **Step 2: Verify** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "from orcapod.protocols.hashing_protocols import SemanticHasherProtocol; print('OK')" - ``` - Expected: `OK` - ---- - -## Task 9: Update `contexts/data/v0.1.json` - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Apply renames** - - Changes needed (4 renames): - 1. Top-level key `"python_type_semantic_hasher_registry"` → `"python_type_handler_registry"` - 2. All `"_class"` values with `*SemanticHasher` suffix — e.g.: - - `"...builtin_handlers.BytesSemanticHasher"` → `"...builtin_handlers.BytesHandler"` - - `"...builtin_handlers.PathSemanticHasher"` → `"...builtin_handlers.PathHandler"` - - `"...builtin_handlers.UPathSemanticHasher"` → `"...builtin_handlers.UPathHandler"` - - `"...builtin_handlers.UUIDSemanticHasher"` → `"...builtin_handlers.UUIDHandler"` - - `"...builtin_handlers.FunctionSemanticHasher"` → `"...builtin_handlers.FunctionHandler"` - - `"...builtin_handlers.TypeObjectSemanticHasher"` → `"...builtin_handlers.TypeObjectHandler"` - - `"...builtin_handlers.GenericAliasSemanticHasher"` → `"...builtin_handlers.GenericAliasHandler"` - - `"...builtin_handlers.UnionTypeSemanticHasher"` → `"...builtin_handlers.UnionTypeHandler"` - - `"...builtin_handlers.SpecialFormSemanticHasher"` → `"...builtin_handlers.SpecialFormHandler"` - - `"...builtin_handlers.ArrowTableSemanticHasher"` → `"...builtin_handlers.ArrowTableHandler"` - - `"...type_handler_registry.PythonTypeSemanticHasherRegistry"` → `"...type_handler_registry.PythonTypeHandlerRegistry"` - 3. Inside `semantic_hasher._config`: sub-key `"type_semantic_hasher_registry"` → `"type_handler_registry"` - 4. Inside `semantic_hasher._config.type_handler_registry`: `"_ref": "python_type_semantic_hasher_registry"` → `"_ref": "python_type_handler_registry"` - -- [ ] **Step 2: Verify JSON is valid and context loads** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "import json; json.load(open('src/orcapod/contexts/data/v0.1.json')); print('JSON OK')" - uv run python -c "from orcapod.contexts import get_default_context; ctx = get_default_context(); print('Context OK')" - ``` - Expected: `JSON OK` then `Context OK` - ---- - -## Task 10: Update `contexts/data/schemas/context_schema.json` - -**Files:** -- Modify: `src/orcapod/contexts/data/schemas/context_schema.json` - -- [ ] **Step 1: Apply renames** - - Changes needed: - - Property key `"python_type_semantic_hasher_registry"` → `"python_type_handler_registry"` (in `properties` section) - - Description string within that property: `"ObjectSpec for the PythonTypeSemanticHasherRegistry..."` → `"ObjectSpec for the PythonTypeHandlerRegistry..."` - - In the `examples` section: `"type_semantic_hasher_registry"` sub-key → `"type_handler_registry"`, and `"_ref": "python_type_semantic_hasher_registry"` → `"_ref": "python_type_handler_registry"` - -- [ ] **Step 2: Verify JSON is valid** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -c "import json; json.load(open('src/orcapod/contexts/data/schemas/context_schema.json')); print('Schema JSON OK')" - ``` - Expected: `Schema JSON OK` - ---- - -## Task 11: Update test files - -**Files:** -- Modify: `tests/test_hashing/test_semantic_hasher.py` -- Modify: `tests/test_hashing/test_uuid_handler.py` -- Modify: `test-objective/unit/test_hashing.py` - -- [ ] **Step 1: Update `tests/test_hashing/test_semantic_hasher.py`** - - Changes needed: - - Import: `register_builtin_python_type_semantic_hashers` → `register_builtin_python_type_handlers` - - Import: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - Import: `get_default_python_type_semantic_hasher_registry` → `get_default_python_type_handler_registry` - - `make_hasher` body: `registry = PythonTypeSemanticHasherRegistry()` → `PythonTypeHandlerRegistry()`, `register_builtin_python_type_semantic_hashers(registry)` → `register_builtin_python_type_handlers(registry)`, `type_semantic_hasher_registry=registry` → `type_handler_registry=registry` - - All other usages of these names throughout the file (type annotations, variable names, docstrings, comments) - -- [ ] **Step 2: Update `tests/test_hashing/test_uuid_handler.py`** - - Changes needed: - - Import: `register_builtin_python_type_semantic_hashers` → `register_builtin_python_type_handlers` - - Import: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - `_make_hasher` body: same pattern as above - - `type_semantic_hasher_registry=registry` → `type_handler_registry=registry` - -- [ ] **Step 3: Update `test-objective/unit/test_hashing.py`** - - Changes needed (this file has many occurrences — all follow the same pattern): - - Imports: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry`, `BuiltinPythonTypeSemanticHasherRegistry` → `BuiltinPythonTypeHandlerRegistry` - - All fixture/function type annotations: `PythonTypeSemanticHasherRegistry` → `PythonTypeHandlerRegistry` - - All constructor calls: `type_semantic_hasher_registry=registry` → `type_handler_registry=registry` - - All class names in test bodies: `PythonTypeSemanticHasherRegistry()` → `PythonTypeHandlerRegistry()` - - All `BuiltinPythonTypeSemanticHasherRegistry()` → `BuiltinPythonTypeHandlerRegistry()` - - All comments/docstrings mentioning old names - -- [ ] **Step 4: Verify test files parse** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run python -m py_compile tests/test_hashing/test_semantic_hasher.py && echo "OK" - uv run python -m py_compile tests/test_hashing/test_uuid_handler.py && echo "OK" - uv run python -m py_compile test-objective/unit/test_hashing.py && echo "OK" - ``` - Expected: three `OK` lines - ---- - -## Task 12: Run tests and commit - -- [ ] **Step 1: Run hashing tests** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run pytest tests/test_hashing/ -x -q - ``` - Expected: all tests pass - -- [ ] **Step 2: Run full test suite (excluding deleted semantic types)** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - uv run pytest tests/ -x -q --ignore=tests/test_semantic_types - ``` - Expected: all tests pass - -- [ ] **Step 3: Confirm no remaining old names in source** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - grep -rn "PathSemanticHasher\|UPathSemanticHasher\|UUIDSemanticHasher\|BytesSemanticHasher\|FunctionSemanticHasher\|TypeObjectSemanticHasher\|SpecialFormSemanticHasher\|GenericAliasSemanticHasher\|UnionTypeSemanticHasher\|ArrowTableSemanticHasher\|SchemaSemanticHasher\|PythonTypeSemanticHasherRegistry\|BuiltinPythonTypeSemanticHasherRegistry\|get_default_python_type_semantic_hasher_registry\|register_builtin_python_type_semantic_hashers\|type_semantic_hasher_registry" src/ tests/ test-objective/ --include="*.py" --include="*.json" | grep -v "^Binary" - ``` - Expected: no matches (zero lines) - -- [ ] **Step 4: Commit** - - ```bash - cd /home/kurouto/kurouto-jobs/dc15d84f-7281-48b5-9e17-435e9a04f175/orcapod-python - git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py - git add src/orcapod/hashing/semantic_hashing/type_handler_registry.py - git add src/orcapod/hashing/semantic_hashing/semantic_hasher.py - git add src/orcapod/hashing/semantic_hashing/__init__.py - git add src/orcapod/hashing/__init__.py - git add src/orcapod/hashing/defaults.py - git add src/orcapod/hashing/versioned_hashers.py - git add src/orcapod/protocols/hashing_protocols.py - git add src/orcapod/contexts/data/v0.1.json - git add src/orcapod/contexts/data/schemas/context_schema.json - git add tests/test_hashing/test_semantic_hasher.py - git add tests/test_hashing/test_uuid_handler.py - git add test-objective/unit/test_hashing.py - git add superpowers/plans/2026-06-24-rename-semantic-hasher-to-handler.md - git commit -m "refactor(hashing): rename *SemanticHasher → *Handler, PythonTypeSemanticHasherRegistry → PythonTypeHandlerRegistry" - ``` diff --git a/superpowers/plans/2026-06-25-plt-1663-merge-extension-type-system-into-main.md b/superpowers/plans/2026-06-25-plt-1663-merge-extension-type-system-into-main.md deleted file mode 100644 index 12f8d3f36..000000000 --- a/superpowers/plans/2026-06-25-plt-1663-merge-extension-type-system-into-main.md +++ /dev/null @@ -1,41 +0,0 @@ -# PLT-1663: Merge extension-type-system → main - -## Overview - -The `extension-type-system` integration branch contains all work from PLT-1652 through -PLT-1660, PLT-1668, and PLT-1672. This plan covers the final steps to bring the branch -up-to-date with `main` and create the merge PR. - -## Situation - -- `extension-type-system` is 205 commits ahead of `main` -- It is 5 commits **behind** `main` (all PLT-1773: pyspiral `0.11.7 → 0.14.9` upgrade) -- The missing commits cause `spiral-integration` CI to fail (external service issue, - not a code bug — fixed by the pyspiral version bump on main) -- All other CI checks pass (unit tests 3.11/3.12, license check) -- Code audit: all old naming patterns removed from production code - - `ExtensionTypeConverter` — gone ✅ - - `ExtensionTypeRegistry` — gone ✅ - - `SemanticTypeRegistry` — only in v0.1.json changelog comment ✅ - - `BaseSemanticHasher` — only in v0.1.json changelog comment ✅ - - Shape-based code — only in explanatory comments ✅ - -## Steps - -1. **Rebase** `extension-type-system` onto `origin/main` - - Brings in 5 PLT-1773 commits (pyspiral fix + lock file updates) - - No conflicts expected (verified via dry-run) - - Will fix the `spiral-integration` CI failure - -2. **Force-push** `extension-type-system` to origin - - Required after rebase; targets feature branch only (not main) - -3. **Create PR** `extension-type-system` → `main` - - Comprehensive description listing all sub-issues resolved - - References PLT-1663 and all related issues (PLT-1652 through PLT-1660, PLT-1668, PLT-1672) - -## Success Criteria - -- CI passes on the updated `extension-type-system` branch -- PR is open and ready for review -- PR description references PLT-1663 and all sub-issues diff --git a/superpowers/plans/2026-06-28-pick-index-operators.md b/superpowers/plans/2026-06-28-pick-index-operators.md deleted file mode 100644 index c26b82a6c..000000000 --- a/superpowers/plans/2026-06-28-pick-index-operators.md +++ /dev/null @@ -1,1558 +0,0 @@ -# Pick and Index Operators Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `Pick` and `Index` operators that let users project into dict/struct and list columns on a stream, with static type resolution and per-packet miss handling. - -**Architecture:** Two `UnaryOperator` subclasses (one per operator) backed by a shared `BaseLogicalType` class that adds `pick_field`/`index_element` to the extension type hierarchy. Both operators use Python-level column access via `data[col]` for both streaming (`async_execute`) and barrier (`unary_static_process`) paths — the stream's type converter handles Arrow ↔ Python round-tripping transparently. - -**Tech Stack:** Python 3.11+, PyArrow, orcapod `UnaryOperator`, `Data.update()` / `Data.with_columns()` / `Data.with_source_info()`, `_materialize_to_stream()`. - ---- - -## File Map - -| Action | Path | Responsibility | -|---|---|---| -| Create | `src/orcapod/extension_types/base_logical_type.py` | `BaseLogicalType` with default `NotImplementedError` impls | -| Modify | `src/orcapod/extension_types/protocols.py` | Add `pick_field`/`index_element` to `LogicalTypeProtocol` | -| Modify | `src/orcapod/extension_types/builtin_logical_types.py` | Inherit `BaseLogicalType` on 3 classes | -| Modify | `src/orcapod/extension_types/dataclass_logical_type_factory.py` | Inherit `BaseLogicalType` on `DataclassLogicalType` | -| Modify | `src/orcapod/extension_types/pydantic_logical_type_factory.py` | Inherit `BaseLogicalType` on `PydanticLogicalType` | -| Create | `src/orcapod/core/operators/pick.py` | `Pick` `UnaryOperator` | -| Create | `src/orcapod/core/operators/index.py` | `Index` `UnaryOperator` | -| Modify | `src/orcapod/core/operators/__init__.py` | Export `Pick`, `Index` | -| Modify | `src/orcapod/operators/__init__.py` | Re-export `Pick`, `Index` | -| Modify | `src/orcapod/core/streams/base.py` | Add `.pick()` / `.index()` convenience methods | -| Create | `tests/test_core/operators/test_pick_index.py` | All unit + integration tests | - ---- - -## Task 1: `BaseLogicalType` + `LogicalTypeProtocol` update - -**Files:** -- Create: `src/orcapod/extension_types/base_logical_type.py` -- Modify: `src/orcapod/extension_types/protocols.py` -- Modify: `src/orcapod/extension_types/builtin_logical_types.py` -- Modify: `src/orcapod/extension_types/dataclass_logical_type_factory.py` -- Modify: `src/orcapod/extension_types/pydantic_logical_type_factory.py` - -- [ ] **Step 1: Write the failing tests for `BaseLogicalType`** - -Create `tests/test_core/operators/test_pick_index.py` with this initial content: - -```python -"""Tests for Pick and Index operators.""" -from __future__ import annotations - -import pytest - -from orcapod.extension_types.base_logical_type import BaseLogicalType - - -class ConcreteLogicalType(BaseLogicalType): - """Minimal concrete subclass for testing defaults.""" - pass - - -def test_base_logical_type_pick_field_raises(): - lt = ConcreteLogicalType() - with pytest.raises(NotImplementedError, match="does not yet support pick"): - lt.pick_field("some_key") - - -def test_base_logical_type_index_element_raises(): - lt = ConcreteLogicalType() - with pytest.raises(NotImplementedError, match="does not yet support index"): - lt.index_element() -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py -v -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.extension_types.base_logical_type'` - -- [ ] **Step 3: Create `BaseLogicalType`** - -Create `src/orcapod/extension_types/base_logical_type.py`: - -```python -"""Base class for all orcapod logical types.""" -from __future__ import annotations - - -class BaseLogicalType: - """Shared base for all logical types. - - Provides default ``NotImplementedError`` implementations for structural - projection methods. Logical types that support ``pick`` or ``index`` - override these methods. - """ - - def pick_field(self, key: str) -> type: - """Return the Python type of field ``key``. - - Args: - key: Name of the field to project into. - - Returns: - The Python type of the requested field. - - Raises: - NotImplementedError: Until implemented for this logical type. - """ - raise NotImplementedError( - f"{type(self).__name__} does not yet support pick (keyed field access). " - "Support for this extension type is planned for a future issue." - ) - - def index_element(self) -> type: - """Return the Python element type for positional list access. - - Returns: - The Python type of elements in this list-like logical type. - - Raises: - NotImplementedError: Until implemented for this logical type. - """ - raise NotImplementedError( - f"{type(self).__name__} does not yet support index (positional access). " - "Support for this extension type is planned for a future issue." - ) -``` - -- [ ] **Step 4: Run the tests — expect them to pass** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py::test_base_logical_type_pick_field_raises tests/test_core/operators/test_pick_index.py::test_base_logical_type_index_element_raises -v -``` - -Expected: PASS - -- [ ] **Step 5: Add `pick_field` / `index_element` signatures to `LogicalTypeProtocol`** - -In `src/orcapod/extension_types/protocols.py`, find the `LogicalTypeProtocol` class (around line 93) and add these two methods after the existing method signatures: - -```python - def pick_field(self, key: str) -> type: - """Return the Python type of field ``key`` in this structured logical type. - - Args: - key: Name of the field to project into. - - Returns: - The Python type of the requested field. - - Raises: - InputValidationError: If the field does not exist in the type's schema. - NotImplementedError: If this logical type does not support keyed access. - """ - ... - - def index_element(self) -> type: - """Return the Python element type for positional list access. - - Returns: - The Python type of elements in this list-like logical type. - - Raises: - NotImplementedError: If this logical type does not support positional access. - """ - ... -``` - -- [ ] **Step 6: Wire `BaseLogicalType` into all five existing implementers** - -In `src/orcapod/extension_types/builtin_logical_types.py`, change three class declarations: - -```python -# Before: -class LogicalPath: -# After: -class LogicalPath(BaseLogicalType): - -# Before: -class LogicalUPath: -# After: -class LogicalUPath(BaseLogicalType): - -# Before: -class LogicalUUID: -# After: -class LogicalUUID(BaseLogicalType): -``` - -Add the import at the top of the file: -```python -from orcapod.extension_types.base_logical_type import BaseLogicalType -``` - -In `src/orcapod/extension_types/dataclass_logical_type_factory.py`, change: -```python -# Before (line 44): -class DataclassLogicalType: -# After: -class DataclassLogicalType(BaseLogicalType): -``` - -Add the import: -```python -from orcapod.extension_types.base_logical_type import BaseLogicalType -``` - -In `src/orcapod/extension_types/pydantic_logical_type_factory.py`, change: -```python -# Before (line 44): -class PydanticLogicalType: -# After: -class PydanticLogicalType(BaseLogicalType): -``` - -Add the import: -```python -from orcapod.extension_types.base_logical_type import BaseLogicalType -``` - -- [ ] **Step 7: Verify the existing test suite still passes** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all existing tests pass (no regressions). - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/extension_types/base_logical_type.py \ - src/orcapod/extension_types/protocols.py \ - src/orcapod/extension_types/builtin_logical_types.py \ - src/orcapod/extension_types/dataclass_logical_type_factory.py \ - src/orcapod/extension_types/pydantic_logical_type_factory.py \ - tests/test_core/operators/test_pick_index.py -git commit -m "feat(extension_types): add BaseLogicalType with pick_field/index_element stubs (ITL-140)" -``` - ---- - -## Task 2: `Pick` operator - -**Files:** -- Create: `src/orcapod/core/operators/pick.py` -- Test: `tests/test_core/operators/test_pick_index.py` - -### Background — key internal APIs - -**Type detection** (in `validate_unary_input`): -```python -import typing -col_type = data_schema[self.column] # Python type from Schema -origin = typing.get_origin(col_type) # e.g. dict, list, or None -args = typing.get_args(col_type) # e.g. (str, int) for dict[str, int] -``` - -`dict[K, V]` columns are stored in Arrow as `pa.large_list(pa.struct([pa.field("key", K), pa.field("value", V)]))`. The Python type converter handles the dict ↔ Arrow round-trip transparently, so `data[col]` always returns a plain Python `dict`. - -**Updating a Data object** (in `async_execute` and `unary_static_process`): -```python -# Replace value in an EXISTING column (out=None): -new_data = ( - data.update(**{self.column: extracted}) - .with_source_info(**{self.column: new_src}) -) - -# ADD a new column (out='new_name'): -new_data = ( - data.with_columns( - column_types={self.out: self._output_type}, - **{self.out: extracted}, - ).with_source_info(**{self.out: new_src}) -) -``` - -**Source token convention:** -```python -src_token = data.source_info().get(self.column) or "" -new_src = f"{src_token}[{self.key!r}]" if src_token else None -``` - -**Rebuilding a stream in `unary_static_process`:** -```python -rows = [] -for tag, data in stream.iter_data(): - ... - rows.append((tag, new_data)) -return self._materialize_to_stream(rows) # static method on StaticOutputOperatorPod -``` - -- [ ] **Step 1: Write failing tests for `Pick` build-time validation** - -Append to `tests/test_core/operators/test_pick_index.py`: - -```python -import pyarrow as pa -import pytest - -from orcapod.core.operators import Pick -from orcapod.core.streams import ArrowTableStream -from orcapod.errors import InputValidationError - - -# ── fixtures ────────────────────────────────────────────────────────────────── - -@pytest.fixture -def dict_stream() -> ArrowTableStream: - """Stream with 1 tag and 1 dict[str, int] data column.""" - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"animal": "cat", "scores": {"speed": 8, "stealth": 9}}, - {"animal": "dog", "scores": {"speed": 7, "strength": 6}}, - {"animal": "bird", "scores": {"speed": 10, "stealth": 7}}, - ], - python_schema={"animal": str, "scores": dict[str, int]}, - ) - return ArrowTableStream(table, tag_columns=["animal"]) - - -# ── build-time validation tests ─────────────────────────────────────────────── - -def test_pick_missing_column_raises(dict_stream): - with pytest.raises(InputValidationError, match="not found in data schema"): - Pick("nonexistent", "speed")(dict_stream) - - -def test_pick_out_collision_raises(dict_stream): - with pytest.raises(InputValidationError, match="already exists"): - Pick("scores", "speed", out="scores")(dict_stream) -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py::test_pick_missing_column_raises tests/test_core/operators/test_pick_index.py::test_pick_out_collision_raises -v -``` - -Expected: `ImportError` (Pick not yet defined). - -- [ ] **Step 3: Implement `Pick`** - -Create `src/orcapod/core/operators/pick.py`: - -```python -"""Pick operator — keyed projection into dict/struct-typed columns.""" -from __future__ import annotations - -import logging -import typing -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - -from orcapod.channels import ReadableChannel, WritableChannel -from orcapod.core.operators.base import UnaryOperator -from orcapod.errors import InputValidationError -from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol -from orcapod.system_constants import constants -from orcapod.types import ColumnConfig, ContentHash, Schema - -if TYPE_CHECKING: - pass - -logger = logging.getLogger(__name__) - -_MISSING = object() # sentinel for unresolved output type - - -class Pick(UnaryOperator): - """Extract a value from a struct- or dict-typed data column by key. - - For ``dict[K, V]`` columns the lookup is per-packet; if the key is absent - the packet is skipped (or an error is raised when ``fail_on_miss=True``). - - For extension-type struct columns (Pydantic, dataclass) the field - existence is validated at build time and guaranteed at runtime. - - Args: - column: Name of the data column to project into. - key: Dict key or struct field name to extract. - out: Output column name. ``None`` (default) replaces ``column`` - in-place; a string adds a new column alongside the original. - fail_on_miss: If ``True``, raise ``RuntimeError`` when the key is - absent in a packet instead of skipping. Excluded from - ``identity_structure`` — miss-handling does not affect - functional output semantics. See ITL-439. - """ - - def __init__( - self, - column: str, - key: str, - out: str | None = None, - fail_on_miss: bool = False, - **kwargs: Any, - ) -> None: - self.column = column - self.key = key - self.out = out - self.fail_on_miss = fail_on_miss - self._output_type: type = _MISSING # type: ignore[assignment] - self._mode: str = "" - super().__init__(**kwargs) - - def identity_structure(self) -> Any: - # fail_on_miss intentionally excluded — it controls error behaviour, - # not the functional transformation. - return (self.__class__.__name__, self.column, self.key, self.out) - - # ------------------------------------------------------------------ - # Validation - # ------------------------------------------------------------------ - - def validate_unary_input(self, stream: StreamProtocol) -> None: - """Validate input schema and resolve output type. - - Args: - stream: The upstream stream to validate. - - Raises: - InputValidationError: If ``column`` is missing, ``out`` collides - with an existing column, or the column type does not support - pick. - NotImplementedError: If the column holds an extension type whose - ``pick_field`` implementation is not yet available. - """ - _, data_schema = stream.output_schema() - data_columns = list(data_schema.keys()) - - if self.column not in data_columns: - raise InputValidationError( - f"Pick: column {self.column!r} not found in data schema. " - f"Available data columns: {data_columns}" - ) - - out_name = self.out if self.out is not None else self.column - if self.out is not None and self.out in data_columns: - raise InputValidationError( - f"Pick: out column {self.out!r} already exists in data schema. " - f"Choose a different name to avoid clobbering existing data." - ) - - col_type = data_schema[self.column] - origin = typing.get_origin(col_type) - - if origin is dict: - # dict[K, V] — dynamic key lookup at runtime - args = typing.get_args(col_type) - self._output_type = args[1] if args else Any # type: ignore[assignment] - self._mode = "map" - else: - # Extension type — delegate to logical type's pick_field - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - # Access registry via the concrete converter (private but stable) - registry = getattr(converter, "_logical_type_registry", None) - if registry is None: - raise InputValidationError( - f"Pick: cannot resolve logical type for column {self.column!r} " - f"(no LogicalTypeRegistry configured)." - ) - lt = registry.get_by_python_type(col_type) - if lt is None: - raise InputValidationError( - f"Pick: column {self.column!r} has type {col_type!r} which is " - f"not a supported pick target (not dict[K,V] and no registered " - f"logical type)." - ) - # May raise NotImplementedError for types not yet implemented - self._output_type = lt.pick_field(self.key) - self._mode = "struct" - - # ------------------------------------------------------------------ - # Schema prediction - # ------------------------------------------------------------------ - - def unary_output_schema( - self, - stream: StreamProtocol, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return the (tag, data) schemas for the output stream. - - Args: - stream: The upstream stream. - columns: Column inclusion config passed through to ``output_schema``. - all_info: Whether to include all info columns. - - Returns: - A ``(tag_schema, data_schema)`` tuple. - """ - tag_schema, data_schema = stream.output_schema(columns=columns, all_info=all_info) - data_dict = dict(data_schema) - - if self.out is None: - # Replace existing column type in-place - data_dict[self.column] = self._output_type - else: - # Add new column; source column type unchanged - data_dict[self.out] = self._output_type - - return tag_schema, Schema(data_dict) - - # ------------------------------------------------------------------ - # Barrier-mode execution - # ------------------------------------------------------------------ - - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: - """Process the full stream in barrier mode. - - Args: - stream: The fully materialised input stream. - - Returns: - A new stream with the projection applied. - - Raises: - RuntimeError: If ``fail_on_miss=True`` and any packet's dict - lacks ``self.key``. - """ - out_rows: list[tuple[Any, Any]] = [] - skipped_count = 0 - - for tag, data in stream.iter_data(): - col_val = data[self.column] - - if self._mode == "struct": - extracted = col_val[self.key] - else: # map - if self.key not in col_val: - skipped_count += 1 - continue - extracted = col_val[self.key] - - src_token = data.source_info().get(self.column) or "" - new_src = f"{src_token}[{self.key!r}]" if src_token else None - - if self.out is None: - new_data = ( - data.update(**{self.column: extracted}) - .with_source_info(**{self.column: new_src}) - ) - else: - new_data = ( - data.with_columns( - column_types={self.out: self._output_type}, - **{self.out: extracted}, - ).with_source_info(**{self.out: new_src}) - ) - - out_rows.append((tag, new_data)) - - if skipped_count: - if self.fail_on_miss: - raise RuntimeError( - f"Pick: {skipped_count} packet(s) missing key {self.key!r} in " - f"column {self.column!r} (fail_on_miss=True). See ITL-439." - ) - logger.warning( - "Pick: %d packet(s) skipped — key %r not found in column %r.", - skipped_count, - self.key, - self.column, - ) - - if not out_rows: - raise ValueError( - f"Pick operator produced an empty stream: all packets were skipped " - f"(key {self.key!r} absent in every packet of column {self.column!r})." - ) - - return self._materialize_to_stream(out_rows) - - # ------------------------------------------------------------------ - # Streaming execution - # ------------------------------------------------------------------ - - async def async_execute( - self, - inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - *, - input_pipeline_hashes: Sequence[ContentHash] | None = None, - ) -> None: - """Process packets one at a time as they arrive. - - Args: - inputs: Single-element sequence of readable channels. - output: Channel to send transformed packets to. - input_pipeline_hashes: Ignored; present for protocol compliance. - """ - try: - async for tag, data in inputs[0]: - col_val = data[self.column] - - if self._mode == "struct": - extracted = col_val[self.key] - else: # map - if self.key not in col_val: - if self.fail_on_miss: - raise RuntimeError( - f"Pick: key {self.key!r} not found in column " - f"{self.column!r} (fail_on_miss=True). See ITL-439." - ) - logger.warning( - "Pick: skipping packet — key %r not found in column %r.", - self.key, - self.column, - ) - continue - extracted = col_val[self.key] - - src_token = data.source_info().get(self.column) or "" - new_src = f"{src_token}[{self.key!r}]" if src_token else None - - if self.out is None: - new_data = ( - data.update(**{self.column: extracted}) - .with_source_info(**{self.column: new_src}) - ) - else: - new_data = ( - data.with_columns( - column_types={self.out: self._output_type}, - **{self.out: extracted}, - ).with_source_info(**{self.out: new_src}) - ) - - await output.send((tag, new_data)) - finally: - await output.close() -``` - -- [ ] **Step 4: Run the build-time validation tests** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py::test_pick_missing_column_raises tests/test_core/operators/test_pick_index.py::test_pick_out_collision_raises -v -``` - -Expected: PASS - -- [ ] **Step 5: Write functional tests for `Pick`** - -Append to `tests/test_core/operators/test_pick_index.py`: - -```python -# ── functional tests: Pick ──────────────────────────────────────────────────── - -def test_pick_dict_default_out(dict_stream): - """pick with out=None replaces the column in-place, source token updated.""" - result = Pick("scores", "speed")(dict_stream) - tag_schema, data_schema = result.output_schema() - - assert "scores" in data_schema - assert data_schema["scores"] == int - - rows = list(result.iter_data()) - assert len(rows) == 3 - values = [data["scores"] for _, data in rows] - assert values == [8, 7, 10] - - # source token for 'scores' should now end with "['speed']" - for _, data in rows: - src = data.source_info().get("scores") - assert src is not None and src.endswith("['speed']"), f"unexpected source: {src}" - - -def test_pick_dict_explicit_out(dict_stream): - """pick with out='speed_score' adds new column, original unchanged.""" - result = Pick("scores", "speed", out="speed_score")(dict_stream) - tag_schema, data_schema = result.output_schema() - - assert "scores" in data_schema # original preserved - assert "speed_score" in data_schema - assert data_schema["speed_score"] == int - - rows = list(result.iter_data()) - assert [data["speed_score"] for _, data in rows] == [8, 7, 10] - - for _, data in rows: - src = data.source_info().get("speed_score") - assert src is not None and src.endswith("['speed']") - - -def test_pick_dict_all_keys_present(dict_stream): - """When all packets contain the key, all pass through with no warning.""" - import logging - with pytest.warns(None) as warning_list: - result = Pick("scores", "speed")(dict_stream) - rows = list(result.iter_data()) - assert len(rows) == 3 - - -def test_pick_dict_missing_key_skip(): - """Packets missing the key are skipped; others pass through.""" - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "data": {"x": 10}}, - {"id": 2, "data": {"y": 20}}, # missing "x" - {"id": 3, "data": {"x": 30}}, - ], - python_schema={"id": int, "data": dict[str, int]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - result = Pick("data", "x", fail_on_miss=False)(stream) - rows = list(result.iter_data()) - assert len(rows) == 2 - ids = [tag["id"] for tag, _ in rows] - assert ids == [1, 3] - - -def test_pick_dict_missing_key_fail(): - """fail_on_miss=True raises RuntimeError when key is absent.""" - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "data": {"x": 10}}, - {"id": 2, "data": {"y": 20}}, - ], - python_schema={"id": int, "data": dict[str, int]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - with pytest.raises(RuntimeError, match="fail_on_miss=True"): - list(Pick("data", "x", fail_on_miss=True)(stream).iter_data()) - - -def test_pick_extension_type_not_implemented(): - """pick on a Pydantic/dataclass column raises NotImplementedError.""" - import dataclasses - - @dataclasses.dataclass - class MyModel: - value: int - - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [{"id": 1, "rec": MyModel(value=42)}], - python_schema={"id": int, "rec": MyModel}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - with pytest.raises(NotImplementedError, match="does not yet support pick"): - Pick("rec", "value")(stream) -``` - -- [ ] **Step 6: Run the functional tests** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py -k "pick" -v -``` - -Expected: all pick tests PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/operators/pick.py tests/test_core/operators/test_pick_index.py -git commit -m "feat(operators): add Pick operator with dict/struct projection (ITL-140)" -``` - ---- - -## Task 3: `Index` operator - -**Files:** -- Create: `src/orcapod/core/operators/index.py` -- Test: `tests/test_core/operators/test_pick_index.py` - -### Background - -`list[T]` columns are stored as `pa.large_list(T_arrow)`. `data[col]` returns a Python list. Negative indices follow Python semantics; bounds check: - -```python -length = len(col_val) -effective_i = self.i if self.i >= 0 else length + self.i -if effective_i < 0 or effective_i >= length: - # out of bounds -``` - -Source token: `f"{src_token}[{self.i}]"` (e.g., `"src::rec::col[3]"`). - -- [ ] **Step 1: Write failing tests for `Index`** - -Append to `tests/test_core/operators/test_pick_index.py`: - -```python -# ── fixtures ────────────────────────────────────────────────────────────────── - -@pytest.fixture -def list_stream() -> ArrowTableStream: - """Stream with 1 tag and 1 list[int] data column.""" - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"animal": "cat", "scores": [8, 5, 9]}, - {"animal": "dog", "scores": [7, 6, 4]}, - {"animal": "bird", "scores": [10, 7, 3]}, - ], - python_schema={"animal": str, "scores": list[int]}, - ) - return ArrowTableStream(table, tag_columns=["animal"]) - - -# ── build-time validation ───────────────────────────────────────────────────── - -def test_index_missing_column_raises(list_stream): - with pytest.raises(InputValidationError, match="not found in data schema"): - from orcapod.core.operators.index import Index - Index("nonexistent", 0)(list_stream) - - -def test_index_out_collision_raises(list_stream): - from orcapod.core.operators.index import Index - with pytest.raises(InputValidationError, match="already exists"): - Index("scores", 0, out="scores")(list_stream) -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py::test_index_missing_column_raises tests/test_core/operators/test_pick_index.py::test_index_out_collision_raises -v -``` - -Expected: `ImportError` (Index not yet defined). - -- [ ] **Step 3: Implement `Index`** - -Create `src/orcapod/core/operators/index.py`: - -```python -"""Index operator — positional projection into list-typed columns.""" -from __future__ import annotations - -import logging -import typing -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - -from orcapod.channels import ReadableChannel, WritableChannel -from orcapod.core.operators.base import UnaryOperator -from orcapod.errors import InputValidationError -from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol -from orcapod.system_constants import constants -from orcapod.types import ColumnConfig, ContentHash, Schema - -logger = logging.getLogger(__name__) - -_MISSING = object() - - -class Index(UnaryOperator): - """Extract an element from a list-typed data column by position. - - The list length is per-packet data, so bounds are not checked at build - time. Out-of-bounds access causes the packet to be skipped (or an error - when ``fail_on_miss=True``). Negative indices follow Python semantics - (``-1`` is the last element). - - Args: - column: Name of the data column to project into. - i: Position to extract. Negative indices follow Python semantics. - out: Output column name. ``None`` (default) replaces ``column`` - in-place; a string adds a new column alongside the original. - fail_on_miss: If ``True``, raise ``RuntimeError`` on out-of-bounds - instead of skipping. Excluded from ``identity_structure``. - See ITL-439. - """ - - def __init__( - self, - column: str, - i: int, - out: str | None = None, - fail_on_miss: bool = False, - **kwargs: Any, - ) -> None: - self.column = column - self.i = i - self.out = out - self.fail_on_miss = fail_on_miss - self._output_type: type = _MISSING # type: ignore[assignment] - self._mode: str = "" - super().__init__(**kwargs) - - def identity_structure(self) -> Any: - return (self.__class__.__name__, self.column, self.i, self.out) - - # ------------------------------------------------------------------ - # Validation - # ------------------------------------------------------------------ - - def validate_unary_input(self, stream: StreamProtocol) -> None: - """Validate input schema and resolve output element type. - - Args: - stream: The upstream stream to validate. - - Raises: - InputValidationError: If ``column`` is missing, ``out`` collides, - or the column type is not a supported index target. - NotImplementedError: If the column holds an extension type whose - ``index_element`` is not yet implemented. - """ - _, data_schema = stream.output_schema() - data_columns = list(data_schema.keys()) - - if self.column not in data_columns: - raise InputValidationError( - f"Index: column {self.column!r} not found in data schema. " - f"Available data columns: {data_columns}" - ) - - if self.out is not None and self.out in data_columns: - raise InputValidationError( - f"Index: out column {self.out!r} already exists in data schema. " - f"Choose a different name to avoid clobbering existing data." - ) - - col_type = data_schema[self.column] - origin = typing.get_origin(col_type) - - if origin is list: - args = typing.get_args(col_type) - self._output_type = args[0] if args else Any # type: ignore[assignment] - self._mode = "list" - else: - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - registry = getattr(converter, "_logical_type_registry", None) - if registry is None: - raise InputValidationError( - f"Index: cannot resolve logical type for column {self.column!r}." - ) - lt = registry.get_by_python_type(col_type) - if lt is None: - raise InputValidationError( - f"Index: column {self.column!r} has type {col_type!r} which is " - f"not a supported index target (not list[T] and no registered " - f"logical type)." - ) - self._output_type = lt.index_element() - self._mode = "extension" - - # ------------------------------------------------------------------ - # Schema prediction - # ------------------------------------------------------------------ - - def unary_output_schema( - self, - stream: StreamProtocol, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return the (tag, data) schemas for the output stream. - - Args: - stream: The upstream stream. - columns: Column inclusion config. - all_info: Include all info columns. - - Returns: - A ``(tag_schema, data_schema)`` tuple. - """ - tag_schema, data_schema = stream.output_schema(columns=columns, all_info=all_info) - data_dict = dict(data_schema) - - if self.out is None: - data_dict[self.column] = self._output_type - else: - data_dict[self.out] = self._output_type - - return tag_schema, Schema(data_dict) - - # ------------------------------------------------------------------ - # Barrier-mode execution - # ------------------------------------------------------------------ - - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: - """Process the full stream in barrier mode. - - Args: - stream: The fully materialised input stream. - - Returns: - A new stream with the projection applied. - - Raises: - RuntimeError: If ``fail_on_miss=True`` and any packet is - out-of-bounds. - """ - out_rows: list[tuple[Any, Any]] = [] - skipped_count = 0 - - for tag, data in stream.iter_data(): - col_val = data[self.column] - length = len(col_val) - effective_i = self.i if self.i >= 0 else length + self.i - - if effective_i < 0 or effective_i >= length: - skipped_count += 1 - continue - - extracted = col_val[self.i] - - src_token = data.source_info().get(self.column) or "" - new_src = f"{src_token}[{self.i}]" if src_token else None - - if self.out is None: - new_data = ( - data.update(**{self.column: extracted}) - .with_source_info(**{self.column: new_src}) - ) - else: - new_data = ( - data.with_columns( - column_types={self.out: self._output_type}, - **{self.out: extracted}, - ).with_source_info(**{self.out: new_src}) - ) - - out_rows.append((tag, new_data)) - - if skipped_count: - if self.fail_on_miss: - raise RuntimeError( - f"Index: {skipped_count} packet(s) out-of-bounds at index " - f"{self.i} in column {self.column!r} (fail_on_miss=True). " - f"See ITL-439." - ) - logger.warning( - "Index: %d packet(s) skipped — index %d out of bounds in column %r.", - skipped_count, - self.i, - self.column, - ) - - if not out_rows: - raise ValueError( - f"Index operator produced an empty stream: all packets were skipped " - f"(index {self.i} out of bounds for every packet in column " - f"{self.column!r})." - ) - - return self._materialize_to_stream(out_rows) - - # ------------------------------------------------------------------ - # Streaming execution - # ------------------------------------------------------------------ - - async def async_execute( - self, - inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - *, - input_pipeline_hashes: Sequence[ContentHash] | None = None, - ) -> None: - """Process packets one at a time as they arrive. - - Args: - inputs: Single-element sequence of readable channels. - output: Channel to send transformed packets to. - input_pipeline_hashes: Ignored; present for protocol compliance. - """ - try: - async for tag, data in inputs[0]: - col_val = data[self.column] - length = len(col_val) - effective_i = self.i if self.i >= 0 else length + self.i - - if effective_i < 0 or effective_i >= length: - if self.fail_on_miss: - raise RuntimeError( - f"Index: index {self.i} out of bounds for column " - f"{self.column!r} (length {length}, " - f"fail_on_miss=True). See ITL-439." - ) - logger.warning( - "Index: skipping packet — index %d out of bounds for " - "column %r (length %d).", - self.i, - self.column, - length, - ) - continue - - extracted = col_val[self.i] - - src_token = data.source_info().get(self.column) or "" - new_src = f"{src_token}[{self.i}]" if src_token else None - - if self.out is None: - new_data = ( - data.update(**{self.column: extracted}) - .with_source_info(**{self.column: new_src}) - ) - else: - new_data = ( - data.with_columns( - column_types={self.out: self._output_type}, - **{self.out: extracted}, - ).with_source_info(**{self.out: new_src}) - ) - - await output.send((tag, new_data)) - finally: - await output.close() -``` - -- [ ] **Step 4: Run the build-time validation tests** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py::test_index_missing_column_raises tests/test_core/operators/test_pick_index.py::test_index_out_collision_raises -v -``` - -Expected: PASS - -- [ ] **Step 5: Write functional tests for `Index`** - -Append to `tests/test_core/operators/test_pick_index.py`: - -```python -# ── functional tests: Index ─────────────────────────────────────────────────── - -def test_index_list_default_out(list_stream): - """index with out=None replaces the column, source token updated.""" - from orcapod.core.operators.index import Index - result = Index("scores", 0)(list_stream) - tag_schema, data_schema = result.output_schema() - - assert "scores" in data_schema - assert data_schema["scores"] == int - - rows = list(result.iter_data()) - assert [data["scores"] for _, data in rows] == [8, 7, 10] - - for _, data in rows: - src = data.source_info().get("scores") - assert src is not None and src.endswith("[0]"), f"unexpected source: {src}" - - -def test_index_list_explicit_out(list_stream): - """index with out='first' adds new column, original unchanged.""" - from orcapod.core.operators.index import Index - result = Index("scores", 0, out="first")(list_stream) - tag_schema, data_schema = result.output_schema() - - assert "scores" in data_schema # original preserved - assert "first" in data_schema - assert data_schema["first"] == int - - rows = list(result.iter_data()) - assert [data["first"] for _, data in rows] == [8, 7, 10] - - for _, data in rows: - src = data.source_info().get("first") - assert src is not None and src.endswith("[0]") - - -def test_index_in_bounds_negative(list_stream): - """i=-1 returns the last element of each list.""" - from orcapod.core.operators.index import Index - result = Index("scores", -1)(list_stream) - rows = list(result.iter_data()) - assert [data["scores"] for _, data in rows] == [9, 4, 3] - - -def test_index_oob_positive_skip(): - """Positive OOB index skips the packet.""" - from orcapod.contexts import get_default_type_converter - from orcapod.core.operators.index import Index - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "vals": [10, 20, 30]}, - {"id": 2, "vals": [40]}, # OOB at index 2 - {"id": 3, "vals": [50, 60, 70]}, - ], - python_schema={"id": int, "vals": list[int]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - result = Index("vals", 2, fail_on_miss=False)(stream) - rows = list(result.iter_data()) - assert len(rows) == 2 - assert [tag["id"] for tag, _ in rows] == [1, 3] - - -def test_index_oob_positive_fail(): - """fail_on_miss=True raises RuntimeError on positive OOB.""" - from orcapod.contexts import get_default_type_converter - from orcapod.core.operators.index import Index - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [{"id": 1, "vals": [10]}, {"id": 2, "vals": []}], - python_schema={"id": int, "vals": list[int]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - with pytest.raises(RuntimeError, match="fail_on_miss=True"): - list(Index("vals", 0, fail_on_miss=True)(stream).iter_data()) - - -def test_index_oob_negative_skip(): - """Negative OOB index (e.g. -5 on length-3 list) skips the packet.""" - from orcapod.contexts import get_default_type_converter - from orcapod.core.operators.index import Index - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "vals": [10, 20, 30]}, - {"id": 2, "vals": [40, 50, 60]}, - ], - python_schema={"id": int, "vals": list[int]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - result = Index("vals", -5, fail_on_miss=False)(stream) - rows = list(result.iter_data()) - assert len(rows) == 0 # all skipped, operator raises ValueError for empty stream - # Note: this test should catch ValueError from empty stream or len==0 - - -def test_index_oob_negative_fail(): - """fail_on_miss=True raises RuntimeError on negative OOB.""" - from orcapod.contexts import get_default_type_converter - from orcapod.core.operators.index import Index - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [{"id": 1, "vals": [10, 20, 30]}], - python_schema={"id": int, "vals": list[int]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - with pytest.raises(RuntimeError, match="fail_on_miss=True"): - list(Index("vals", -5, fail_on_miss=True)(stream).iter_data()) -``` - -- [ ] **Step 6: Run all Index tests** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py -k "index" -v -``` - -Expected: all index tests PASS. Note: `test_index_oob_negative_skip` expects a `ValueError` from the empty-stream guard in `unary_static_process` — adjust the assertion if needed. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/operators/index.py tests/test_core/operators/test_pick_index.py -git commit -m "feat(operators): add Index operator with list positional projection (ITL-140)" -``` - ---- - -## Task 4: Wire exports and stream convenience methods - -**Files:** -- Modify: `src/orcapod/core/operators/__init__.py` -- Modify: `src/orcapod/operators/__init__.py` -- Modify: `src/orcapod/core/streams/base.py` -- Test: `tests/test_core/operators/test_pick_index.py` - -- [ ] **Step 1: Write failing import tests** - -Append to `tests/test_core/operators/test_pick_index.py`: - -```python -# ── export / API tests ──────────────────────────────────────────────────────── - -def test_pick_importable_from_public_api(): - from orcapod.operators import Pick # noqa: F401 - - -def test_index_importable_from_public_api(): - from orcapod.operators import Index # noqa: F401 - - -def test_stream_has_pick_method(dict_stream): - assert hasattr(dict_stream, "pick") - result = dict_stream.pick("scores", "speed") - tag_schema, data_schema = result.output_schema() - assert data_schema["scores"] == int - - -def test_stream_has_index_method(list_stream): - assert hasattr(list_stream, "index") - result = list_stream.index("scores", 1) - tag_schema, data_schema = result.output_schema() - assert data_schema["scores"] == int -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py -k "importable or stream_has" -v -``` - -Expected: `ImportError` for Pick/Index from public API, `AttributeError` for stream methods. - -- [ ] **Step 3: Update `src/orcapod/core/operators/__init__.py`** - -```python -from .batch import Batch -from .column_selection import ( - DropDataColumns, - DropTagColumns, - SelectDataColumns, - SelectTagColumns, -) -from .filters import PolarsFilter -from .index import Index -from .join import Join -from .mappers import MapData, MapTags -from .merge_join import MergeJoin -from .pick import Pick -from .semijoin import SemiJoin - -__all__ = [ - "Join", - "MergeJoin", - "SemiJoin", - "MapTags", - "MapData", - "Batch", - "SelectTagColumns", - "SelectDataColumns", - "DropTagColumns", - "DropDataColumns", - "PolarsFilter", - "Pick", - "Index", -] -``` - -- [ ] **Step 4: Update `src/orcapod/operators/__init__.py`** - -Add `Pick` and `Index` to whatever re-exports already exist in that file. Open it, read the existing imports, and add: - -```python -from orcapod.core.operators import Pick, Index -``` - -and include them in `__all__` if one exists. - -- [ ] **Step 5: Add `.pick()` and `.index()` to `StreamBase`** - -In `src/orcapod/core/streams/base.py`, add after the existing convenience methods (e.g., after `batch`): - -```python - def pick( - self, - column: str, - key: str, - out: str | None = None, - fail_on_miss: bool = False, - label: str | None = None, - ) -> "StreamBase": - """Extract a value from a struct- or dict-typed column by key. - - Args: - column: Data column to project into. - key: Field name (struct) or dict key to extract. - out: Output column name. ``None`` replaces ``column`` in-place; - a string adds a new column alongside the original. - fail_on_miss: If ``True``, raise on missing key instead of - skipping. - label: Optional pipeline label for this node. - - Returns: - A new stream with the projection applied. - """ - from orcapod.core.operators import Pick - return Pick(column, key, out=out, fail_on_miss=fail_on_miss)(self, label=label) - - def index( - self, - column: str, - i: int, - out: str | None = None, - fail_on_miss: bool = False, - label: str | None = None, - ) -> "StreamBase": - """Extract an element from a list-typed column by position. - - Args: - column: Data column to project into. - i: Position to extract. Negative indices follow Python semantics - (``-1`` is the last element). - out: Output column name. ``None`` replaces ``column`` in-place; - a string adds a new column alongside the original. - fail_on_miss: If ``True``, raise on out-of-bounds instead of - skipping. - label: Optional pipeline label for this node. - - Returns: - A new stream with the projection applied. - """ - from orcapod.core.operators import Index - return Index(column, i, out=out, fail_on_miss=fail_on_miss)(self, label=label) -``` - -- [ ] **Step 6: Run the export/API tests** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py -k "importable or stream_has" -v -``` - -Expected: PASS - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/operators/__init__.py \ - src/orcapod/operators/__init__.py \ - src/orcapod/core/streams/base.py \ - tests/test_core/operators/test_pick_index.py -git commit -m "feat(operators): export Pick/Index and add .pick()/.index() stream methods (ITL-140)" -``` - ---- - -## Task 5: Integration tests - -**Files:** -- Test: `tests/test_core/operators/test_pick_index.py` - -- [ ] **Step 1: Write integration tests** - -Append to `tests/test_core/operators/test_pick_index.py`: - -```python -# ── integration tests ───────────────────────────────────────────────────────── - -def test_chained_pick_then_index(): - """stream.pick('col', 'key').index('col', 1) chains correctly end-to-end.""" - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "data": {"scores": [10, 20, 30]}}, - {"id": 2, "data": {"scores": [40, 50, 60]}}, - ], - python_schema={"id": int, "data": dict[str, list[int]]}, - ) - stream = ArrowTableStream(table, tag_columns=["id"]) - - result = stream.pick("data", "scores").index("data", 1) - tag_schema, data_schema = result.output_schema() - - assert "data" in data_schema - assert data_schema["data"] == int - - rows = list(result.iter_data()) - assert [data["data"] for _, data in rows] == [20, 50] - - # Source token should encode both projections - for _, data in rows: - src = data.source_info().get("data") - assert src is not None - assert "['scores']" in src - assert "[1]" in src - - -def test_composition_with_join(): - """pick used in a pipeline that also includes join.""" - from orcapod.contexts import get_default_type_converter - converter = get_default_type_converter() - - left_table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "meta": {"label": "alpha"}}, - {"id": 2, "meta": {"label": "beta"}}, - ], - python_schema={"id": int, "meta": dict[str, str]}, - ) - right_table = converter.python_dicts_to_arrow_table( - [ - {"id": 1, "value": 100}, - {"id": 2, "value": 200}, - ], - python_schema={"id": int, "value": int}, - ) - left = ArrowTableStream(left_table, tag_columns=["id"]) - right = ArrowTableStream(right_table, tag_columns=["id"]) - - result = left.pick("meta", "label").join(right) - tag_schema, data_schema = result.output_schema() - - assert "meta" in data_schema # now holds str, not dict - assert "value" in data_schema - assert data_schema["meta"] == str - - rows = list(result.iter_data()) - assert len(rows) == 2 -``` - -- [ ] **Step 2: Run the integration tests** - -```bash -uv run pytest tests/test_core/operators/test_pick_index.py -k "chained or composition" -v -``` - -Expected: PASS - -- [ ] **Step 3: Run the full test suite to verify no regressions** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_core/operators/test_pick_index.py -git commit -m "test(operators): add integration tests for Pick/Index chaining and join (ITL-140)" -``` - ---- - -## Task 6: File follow-up Linear issues and create PR - -- [ ] **Step 1: File two follow-up Linear issues** - -File these two issues in the "Tools" team, "Orcapod Python v0.2 Stabilization Push" project, referencing ITL-140: - -**Issue A — dataclass support:** -- Title: `Add pick_field / index_element to DataclassLogicalType (ITL-140 follow-up)` -- Description (use the project CLAUDE.md template): - - Overview: Implement `pick_field(key)` and `index_element()` on `DataclassLogicalType` so that `Pick` and `Index` operators can project into dataclass-typed columns with static field-name validation and runtime Python-object access. - - Success criteria: `pick` on a `dataclass` column resolves field type at build time; runtime accesses the Python object's attribute; missing field raises `InputValidationError` at build time. - - Reference: ITL-140. - -**Issue B — Pydantic support:** -- Title: `Add pick_field / index_element to PydanticLogicalType (ITL-140 follow-up)` -- Same structure as Issue A but for `PydanticLogicalType`. - -- [ ] **Step 2: Push the branch** - -```bash -git push -u origin eywalker/itl-140-add-pick-and-index-operators-for-structural-projection-into -``` - -- [ ] **Step 3: Create the PR** using `sensei:create-pr` - -Target branch: `main`. Include `Fixes ITL-140` in the description body. diff --git a/superpowers/plans/2026-06-29-itl-45-datetime-tag-values.md b/superpowers/plans/2026-06-29-itl-45-datetime-tag-values.md deleted file mode 100644 index b81515d97..000000000 --- a/superpowers/plans/2026-06-29-itl-45-datetime-tag-values.md +++ /dev/null @@ -1,421 +0,0 @@ -# ITL-45: date/datetime Tag Values Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `datetime.date` and `datetime.datetime` as first-class valid types for Tag values, closing the gap between what the type system declares and what the Arrow converter can actually handle. - -**Architecture:** Three independent layers are updated: (1) the `starfix` dependency is bumped to 0.4.0 so `pa.timestamp` columns can be content-hashed; (2) `UniversalTypeConverter` is extended with a direct `date → pa.date32()` mapping and a `pa.date32() → date` reverse mapping; (3) the public type aliases `TagValue` and `SupportedNativePythonData` are updated to formally include both types. Tests are written first (TDD) to nail down exact expected behavior before touching implementation. - -**Tech Stack:** Python 3.12, PyArrow, starfix 0.4.0, pytest via `uv run pytest` - ---- - -### Task 1: Bump starfix to 0.4.0 - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Update the starfix pin in `pyproject.toml`** - - Find the line: - ``` - "starfix~=0.3.0", - ``` - Replace it with: - ``` - "starfix~=0.4.0", - ``` - -- [ ] **Step 2: Sync the environment** - - ```bash - uv sync - ``` - - Expected: resolves and installs `starfix==0.4.x`. No errors. - -- [ ] **Step 3: Smoke-test that timestamp hashing now works** - - ```bash - uv run python -c " - import pyarrow as pa - from starfix import ArrowDigester - from datetime import datetime, timezone - dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) - t = pa.table({'ts': pa.array([dt], type=pa.timestamp('us', tz='UTC'))}) - h = ArrowDigester.hash_table(t) - print('OK:', h.hex()[:16]) - " - ``` - - Expected: prints `OK:` followed by a hex string. No `NotImplementedError`. - -- [ ] **Step 4: Commit** - - ```bash - git add pyproject.toml uv.lock - git commit -m "chore(deps): bump starfix to ~=0.4.0 for datetime hashing support" - ``` - ---- - -### Task 2: Write failing tests for `date` type conversion - -**Files:** -- Modify: `test-objective/unit/test_semantic_types.py` - -- [ ] **Step 1: Add `date` imports at the top of the test file** - - The file already imports `pa` and `pytest`. Add `date` and `datetime` to the top-level imports: - - ```python - from datetime import date, datetime, timezone - ``` - -- [ ] **Step 2: Add a failing test for `python_type_to_arrow_type(date)`** - - Inside the existing `TestPythonToArrowType` class, add: - - ```python - def test_date_to_date32(self, converter): - result = converter.python_type_to_arrow_type(date) - assert result == pa.date32() - ``` - -- [ ] **Step 3: Add a failing test for `arrow_type_to_python_type(pa.date32())`** - - Inside the existing `TestArrowToPythonType` class, add: - - ```python - def test_date32_to_date(self, converter): - result = converter.arrow_type_to_python_type(pa.date32()) - assert result is date - ``` - -- [ ] **Step 4: Add a failing test for schema round-trip with `date` and `datetime`** - - Inside the existing `TestSchemaConversionRoundtrip` class, add: - - ```python - def test_date_datetime_schema_roundtrip(self, converter): - from orcapod.types import Schema - original = Schema({"dob": date, "ts": datetime}) - arrow_schema = converter.python_schema_to_arrow_schema(original) - recovered = converter.arrow_schema_to_python_schema(arrow_schema) - assert recovered["dob"] is date - assert recovered["ts"] is datetime - ``` - -- [ ] **Step 5: Run the new tests and verify they fail** - - ```bash - uv run pytest test-objective/unit/test_semantic_types.py -k "date" -v - ``` - - Expected: `test_date32_to_date` and `test_date_datetime_schema_roundtrip` FAIL (currently `arrow_type_to_python_type(pa.date32())` returns `typing.Any`). `test_date_to_date32` may already pass via the `__name__` fallback — that's fine, it still documents required behaviour. - ---- - -### Task 3: Implement `date` support in `UniversalTypeConverter` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` - -- [ ] **Step 1: Add `date` to the module-level import** - - Find the existing line near the top of the file: - ```python - from datetime import datetime, timezone - ``` - Replace with: - ```python - from datetime import date, datetime, timezone - ``` - -- [ ] **Step 2: Add `date → pa.date32()` to `_get_python_to_arrow_map()`** - - Inside `_get_python_to_arrow_map()`, find the existing `datetime` entry: - ```python - datetime: pa.timestamp("us", tz="UTC"), - ``` - Add `date` immediately after it: - ```python - datetime: pa.timestamp("us", tz="UTC"), - date: pa.date32(), - ``` - -- [ ] **Step 3: Add `pa.date32() → date` to `_convert_arrow_to_python()`** - - Inside `_convert_arrow_to_python()`, find the existing timestamp branch: - ```python - elif pa.types.is_timestamp(arrow_type): - return datetime - ``` - Add the date branch **before** it: - ```python - elif pa.types.is_date(arrow_type): - return date - elif pa.types.is_timestamp(arrow_type): - return datetime - ``` - -- [ ] **Step 4: Run the tests and verify they all pass** - - ```bash - uv run pytest test-objective/unit/test_semantic_types.py -k "date" -v - ``` - - Expected: all three new tests PASS. Also run the full semantic types suite to check for regressions: - - ```bash - uv run pytest test-objective/unit/test_semantic_types.py -v - ``` - - Expected: all tests PASS. - -- [ ] **Step 5: Commit** - - ```bash - git add src/orcapod/semantic_types/universal_converter.py \ - test-objective/unit/test_semantic_types.py - git commit -m "feat(types): add datetime.date <-> pa.date32() support in UniversalTypeConverter" - ``` - ---- - -### Task 4: Update public type aliases - -**Files:** -- Modify: `src/orcapod/types.py` - -- [ ] **Step 1: Add `date` to the datetime import** - - Find the existing import in `types.py`: - ```python - from datetime import datetime - ``` - Replace with: - ```python - from datetime import date, datetime - ``` - -- [ ] **Step 2: Update `TagValue`** - - Find: - ```python - # TODO: accommodate other common data types such as datetime - TagValue: TypeAlias = int | str | None | Collection["TagValue"] - ``` - Replace with: - ```python - TagValue: TypeAlias = int | str | date | datetime | None | Collection["TagValue"] - ``` - (Remove the TODO comment — it is now resolved.) - - Update its docstring to reflect the new types: - ```python - """A tag metadata value: an int, string, date, datetime, ``None``, or an - arbitrarily nested collection thereof. Tags are used to label and organise - data and datagrams.""" - ``` - -- [ ] **Step 3: Update `SupportedNativePythonData`** - - Find: - ```python - SupportedNativePythonData: TypeAlias = str | int | float | bool | bytes - ``` - Replace with: - ```python - SupportedNativePythonData: TypeAlias = str | int | float | bool | bytes | date | datetime - ``` - - Update its docstring: - ```python - """The simple Python scalar types that have a direct Arrow / Polars - correspondence, including temporal types ``date`` and ``datetime``.""" - ``` - -- [ ] **Step 4: Verify no import or type errors** - - ```bash - uv run python -c "from orcapod.types import TagValue, SupportedNativePythonData; print(TagValue, SupportedNativePythonData)" - ``` - - Expected: prints the two type aliases without error. - -- [ ] **Step 5: Commit** - - ```bash - git add src/orcapod/types.py - git commit -m "feat(types): add date and datetime to TagValue and SupportedNativePythonData" - ``` - ---- - -### Task 5: Write and verify Tag integration tests - -**Files:** -- Modify: `test-objective/unit/test_tag.py` - -- [ ] **Step 1: Add imports at the top of `test_tag.py`** - - The file already imports `pa`, `pytest`, `Tag`, and `ColumnConfig`. Add: - ```python - from datetime import date, datetime, timezone - ``` - -- [ ] **Step 2: Add the new test class** - - Append the following class to the end of `test_tag.py`: - - ```python - # --------------------------------------------------------------------------- - # date and datetime tag values - # --------------------------------------------------------------------------- - - class TestTagDatetimeValues: - """Tags accept date and datetime values and round-trip them correctly.""" - - def test_date_tag_construction(self): - ctx = _make_context() - tag = Tag({"dob": date(2024, 1, 15)}, data_context=ctx) - assert "dob" in tag.keys() - - def test_date_tag_as_dict_roundtrip(self): - ctx = _make_context() - d = date(2024, 1, 15) - tag = Tag({"dob": d}, data_context=ctx) - result = tag.as_dict() - assert result["dob"] == d - assert type(result["dob"]) is date - - def test_date_tag_as_table_schema(self): - ctx = _make_context() - tag = Tag({"dob": date(2024, 1, 15)}, data_context=ctx) - table = tag.as_table() - assert pa.types.is_date(table.schema.field("dob").type) - - def test_date_tag_content_hash(self): - ctx = _make_context() - tag = Tag({"dob": date(2024, 1, 15)}, data_context=ctx) - h = tag.content_hash() - assert h is not None - - def test_datetime_tag_construction(self): - ctx = _make_context() - dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) - tag = Tag({"ts": dt}, data_context=ctx) - assert "ts" in tag.keys() - - def test_datetime_tag_as_dict_roundtrip(self): - ctx = _make_context() - dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) - tag = Tag({"ts": dt}, data_context=ctx) - result = tag.as_dict() - assert result["ts"] == dt - assert type(result["ts"]) is datetime - - def test_datetime_tag_as_table_schema(self): - ctx = _make_context() - dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) - tag = Tag({"ts": dt}, data_context=ctx) - table = tag.as_table() - assert pa.types.is_timestamp(table.schema.field("ts").type) - - def test_datetime_tag_content_hash(self): - ctx = _make_context() - dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) - tag = Tag({"ts": dt}, data_context=ctx) - h = tag.content_hash() - assert h is not None - - def test_naive_datetime_raises(self): - ctx = _make_context() - with pytest.raises(ValueError, match="[Nn]aive datetime"): - tag = Tag({"ts": datetime(2024, 1, 15)}, data_context=ctx) - tag.as_table() # conversion is lazy; trigger it - - def test_date_tag_schema_inference(self): - """Schema inferred from an Arrow-backed Tag preserves date type.""" - ctx = _make_context() - tag = Tag({"dob": date(2024, 1, 15)}, data_context=ctx) - arrow_table = tag.as_table() - tag2 = Tag(arrow_table, data_context=ctx) - assert tag2.schema()["dob"] is date - - def test_datetime_tag_schema_inference(self): - """Schema inferred from an Arrow-backed Tag preserves datetime type.""" - ctx = _make_context() - dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) - tag = Tag({"ts": dt}, data_context=ctx) - arrow_table = tag.as_table() - tag2 = Tag(arrow_table, data_context=ctx) - assert tag2.schema()["ts"] is datetime - ``` - -- [ ] **Step 3: Run the new tests before implementing (verify current failures)** - - ```bash - uv run pytest test-objective/unit/test_tag.py::TestTagDatetimeValues -v - ``` - - Expected failures before Task 3 is complete: - - `test_date_tag_schema_inference` — `Any` instead of `date` - - `test_datetime_tag_schema_inference` — `Any` instead of `datetime` (or may pass if ENG-387 landed) - - `test_datetime_tag_content_hash` — `NotImplementedError` if starfix not yet bumped - - After Tasks 1–4 all complete, re-run: - - ```bash - uv run pytest test-objective/unit/test_tag.py::TestTagDatetimeValues -v - ``` - - Expected: all 11 tests PASS. - -- [ ] **Step 4: Run the full test-objective suite to check for regressions** - - ```bash - uv run pytest test-objective/ -v - ``` - - Expected: all tests PASS. - -- [ ] **Step 5: Run the main test suite** - - ```bash - uv run pytest tests/ -v - ``` - - Expected: all tests PASS. - -- [ ] **Step 6: Commit** - - ```bash - git add test-objective/unit/test_tag.py - git commit -m "test(tag): add date and datetime tag value tests" - ``` - ---- - -### Task 6: Final verification and push - -- [ ] **Step 1: Run the complete test suite one final time** - - ```bash - uv run pytest test-objective/ tests/ -v - ``` - - Expected: all tests PASS, no warnings about `arrow_type_to_python_type` falling back to `Any`. - -- [ ] **Step 2: Check out the feature branch** - - ```bash - git checkout -b eywalker/itl-45-add-support-for-datetime-in-tag-values - ``` - -- [ ] **Step 3: Push the branch** - - ```bash - git push -u origin eywalker/itl-45-add-support-for-datetime-in-tag-values - ``` diff --git a/superpowers/plans/2026-06-29-linear-release-system.md b/superpowers/plans/2026-06-29-linear-release-system.md deleted file mode 100644 index 6b88ab423..000000000 --- a/superpowers/plans/2026-06-29-linear-release-system.md +++ /dev/null @@ -1,317 +0,0 @@ -# Linear Release System Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Wire `linear/linear-release-action` into orcapod-python's CI so that cutting a release tag automatically transitions all Merged Linear issues to Done. - -**Architecture:** Two new workflow jobs (`linear-sync`, `linear-complete`) are added to the existing `publish.yml`; a new `release-sync.yml` mirrors axon's push-to-main sync. No application code changes — this is CI infrastructure only. - -**Tech Stack:** GitHub Actions, `linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f` (v0), `actions/checkout@v4` - ---- - -## File Map - -| Action | Path | What it does | -|--------|------|--------------| -| Create | `.github/workflows/release-sync.yml` | Syncs merged PRs to the open Linear release draft on every push to `main` | -| Modify | `.github/workflows/publish.yml` | Adds `linear-sync` (parallel, tag push) and `linear-complete` (after `publish-pypi`) jobs | -| Modify | `RELEASING.md` | Documents the Linear release system for future maintainers | - ---- - -## Task 1: Create the feature branch - -- [ ] **Step 1.1: Check out the branch** - -```bash -git checkout -b eywalker/itl-448-set-up-linear-release-system-for-orcapod-python-v01-release -``` - -- [ ] **Step 1.2: Verify you are on the correct branch** - -```bash -git branch --show-current -``` - -Expected output: -``` -eywalker/itl-448-set-up-linear-release-system-for-orcapod-python-v01-release -``` - ---- - -## Task 2: Create `release-sync.yml` - -This file mirrors `metamorphic-brain/axon`'s `release-sync.yml` exactly. It runs on every push to `main` and calls `linear-release-action sync` (no `version` argument) to associate the merged PR with the currently-open Linear release draft. - -**Files:** -- Create: `.github/workflows/release-sync.yml` - -- [ ] **Step 2.1: Create the file** - -Write `.github/workflows/release-sync.yml` with this exact content: - -```yaml -name: Linear release sync - -on: - push: - branches: - - main - -jobs: - sync: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Sync Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: sync -``` - -- [ ] **Step 2.2: Validate YAML syntax** - -```bash -python3 -c "import yaml; yaml.safe_load(open('.github/workflows/release-sync.yml')); print('OK')" -``` - -Expected output: `OK` - -- [ ] **Step 2.3: Commit** - -```bash -git add .github/workflows/release-sync.yml -git commit -m "ci: add Linear release-sync workflow on push to main (ITL-448)" -``` - ---- - -## Task 3: Add `linear-sync` and `linear-complete` jobs to `publish.yml` - -`publish.yml` currently ends with the `publish-pypi` job. We append two new jobs after it: - -- **`linear-sync`**: no dependencies (runs in parallel with the existing pipeline), calls `sync` with `version: ${{ github.ref_name }}` to finalise which commits belong to this release in Linear. -- **`linear-complete`**: `needs: [publish-pypi]`, calls `complete` with `version: ${{ github.ref_name }}` to trigger the Merged → Done state transition in Linear. - -**Files:** -- Modify: `.github/workflows/publish.yml` - -- [ ] **Step 3.1: Append the two new jobs to `publish.yml`** - -Open `.github/workflows/publish.yml`. After the closing of the `publish-pypi` job (the last job in the file), append the following at the same indentation level as the other jobs: - -```yaml - - linear-sync: - name: Sync Linear release - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Sync Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: sync - version: ${{ github.ref_name }} - - linear-complete: - name: Complete Linear release - needs: [publish-pypi] - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Complete Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: complete - version: ${{ github.ref_name }} -``` - -- [ ] **Step 3.2: Validate YAML syntax** - -```bash -python3 -c "import yaml; yaml.safe_load(open('.github/workflows/publish.yml')); print('OK')" -``` - -Expected output: `OK` - -- [ ] **Step 3.3: Verify job structure** - -```bash -python3 -c " -import yaml -wf = yaml.safe_load(open('.github/workflows/publish.yml')) -jobs = list(wf['jobs'].keys()) -print('Jobs:', jobs) -assert 'linear-sync' in jobs, 'linear-sync missing' -assert 'linear-complete' in jobs, 'linear-complete missing' -lc = wf['jobs']['linear-complete'] -assert lc['needs'] == ['publish-pypi'], f'wrong needs: {lc[\"needs\"]}' -print('Assertions passed') -" -``` - -Expected output: -``` -Jobs: ['license-check', 'test', 'build', 'publish-testpypi', 'publish-pypi', 'linear-sync', 'linear-complete'] -Assertions passed -``` - -- [ ] **Step 3.4: Commit** - -```bash -git add .github/workflows/publish.yml -git commit -m "ci: add Linear sync and complete jobs to publish workflow (ITL-448)" -``` - ---- - -## Task 4: Update `RELEASING.md` - -Add a new section documenting the Linear release system so future maintainers understand the two-phase flow and know what manual steps are required before the first release. - -**Files:** -- Modify: `RELEASING.md` - -- [ ] **Step 4.1: Append the Linear release section to `RELEASING.md`** - -Open `RELEASING.md` and append the following after the existing content: - -```markdown - -## Linear Release System - -When a release tag is pushed, the CI automatically transitions all Linear issues in -**"Merged"** status (whose PRs were included in the release) to **"Done"**. This is -handled by two GitHub Actions jobs that call `linear/linear-release-action`: - -### How it works - -| Trigger | Job | Action | Effect | -|---------|-----|--------|--------| -| Push to `main` | `release-sync.yml / sync` | `sync` (no version) | Associates the merged PR with the open Linear release draft | -| Tag push `v*` | `publish.yml / linear-sync` | `sync --version ` | Finalises the commit set for this release version in Linear | -| After `publish-pypi` succeeds | `publish.yml / linear-complete` | `complete --version ` | Marks the release done in Linear; triggers Merged → Done | - -### Prerequisites - -Before cutting the first release with this system active, a workspace admin must: - -1. **Set the `LINEAR_ACCESS_KEY` repo secret** in `nauticalab/orcapod-python` GitHub - settings (`Settings → Secrets and variables → Actions → New repository secret`). - Use the same Linear API key as `metamorphic-brain/axon`. - -2. **Configure a Linear release** in the workspace for orcapod-python via the Linear UI: - set the release name, which statuses count as "included" (at minimum: Merged), and - the target state for the transition (Done). - -If `LINEAR_ACCESS_KEY` is not set, the `linear-sync` and `linear-complete` jobs will -fail but will not block the PyPI publish (they have no dependents). - -### Replicating for other orcapod repos - -1. Copy `.github/workflows/release-sync.yml` verbatim. -2. Add `linear-sync` and `linear-complete` jobs to the repo's publish/release workflow, - adjusting `needs:` in `linear-complete` to point at whichever job creates the GitHub - Release. -3. Set the `LINEAR_ACCESS_KEY` repo secret. -4. Configure a Linear release in the workspace for the new repo. -``` - -- [ ] **Step 4.2: Verify the file ends correctly** - -```bash -tail -10 RELEASING.md -``` - -Expected: the last few lines of the replication guide are visible with no truncation. - -- [ ] **Step 4.3: Commit** - -```bash -git add RELEASING.md -git commit -m "docs: document Linear release system in RELEASING.md (ITL-448)" -``` - ---- - -## Task 5: Push branch and open PR - -- [ ] **Step 5.1: Re-authenticate (tokens expire after 1 hour)** - -```bash -gh-app-token-generator nauticalab | gh auth login --with-token -``` - -- [ ] **Step 5.2: Push the branch** - -```bash -git push -u origin eywalker/itl-448-set-up-linear-release-system-for-orcapod-python-v01-release -``` - -- [ ] **Step 5.3: Open the PR** - -```bash -gh pr create \ - --title "ci: set up Linear release system for v0.1 release cut (ITL-448)" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Adds `.github/workflows/release-sync.yml` — mirrors axon's setup; syncs merged PRs to the open Linear release draft on every push to `main`. -- Adds `linear-sync` job to `publish.yml` — runs in parallel on tag push; associates the release tag's commits with the Linear release. -- Adds `linear-complete` job to `publish.yml` — runs after `publish-pypi`; calls `complete` to transition all Merged issues linked to this release to Done in Linear. -- Updates `RELEASING.md` with a new **Linear Release System** section documenting the flow, prerequisites, and replication guide. - -Closes ITL-448 - -## Manual prerequisites (before first release) - -> These steps are **not** automated by this PR and must be completed by a workspace admin: -> -> 1. Set `LINEAR_ACCESS_KEY` as a repo secret in `nauticalab/orcapod-python` GitHub settings (same key as `metamorphic-brain/axon`). -> 2. Configure a Linear release in the workspace for orcapod-python via the Linear UI (release name, included statuses, Merged → Done transition). - -## Test plan - -- [ ] Verify YAML is valid: `python3 -c "import yaml; yaml.safe_load(open('.github/workflows/publish.yml'))"` -- [ ] After `LINEAR_ACCESS_KEY` secret is set: push a test tag (e.g. `v0.0.99-test`) and confirm `linear-sync` and `linear-complete` jobs run green in the Actions tab. -- [ ] Confirm a "Merged" Linear issue linked to a PR in that test release transitions to "Done". - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - -- [ ] **Step 5.4: Report the PR URL** - -The `gh pr create` command above will print the PR URL. Copy it and share with the team. - ---- - -## Post-PR checklist - -- [ ] Remind eywalker to set `LINEAR_ACCESS_KEY` repo secret before cutting v0.1 -- [ ] Confirm Linear workspace has a release configured for orcapod-python (release name, Merged → Done transition) diff --git a/superpowers/plans/2026-06-29-typing-literal-support.md b/superpowers/plans/2026-06-29-typing-literal-support.md deleted file mode 100644 index eaaf34e84..000000000 --- a/superpowers/plans/2026-06-29-typing-literal-support.md +++ /dev/null @@ -1,446 +0,0 @@ -# typing.Literal Support in UniversalTypeConverter — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `typing.Literal` support to `UniversalTypeConverter` so pydantic models with `Literal[...]` fields can be registered and serialized as Arrow pipeline columns. - -**Architecture:** Two surgical branch additions to `universal_converter.py` — one in `_register_python_class_impl` (type-registration path called by the pydantic factory) and one in `_convert_python_to_arrow` (schema-inference/serialization path called during value conversion). No new files. All tests added to the existing pydantic factory test file. - -**Tech Stack:** Python `typing.Literal` / `get_origin` / `get_args`, PyArrow, pydantic v2, pytest. Run all commands via `uv run` (never bare `python` or `pytest`). - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/semantic_types/universal_converter.py` | Modify | `Literal` branch in `_register_python_class_impl` (after line ~295); `Literal` branch in `_convert_python_to_arrow` (after line ~1075) | -| `tests/test_extension_types/test_pydantic_logical_type_factory.py` | Modify | Import `Literal`; 6 new module-level model classes; 7 new test functions | - ---- - -## Task 1: Registration-path tests + fix `_register_python_class_impl` - -**Files:** -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` -- Modify: `src/orcapod/semantic_types/universal_converter.py` - -- [ ] **Step 1.1: Add `Literal` to the import line in the test file** - -Find line 5 in `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -from typing import Any -``` - -Replace with: - -```python -from typing import Any, Literal -``` - -- [ ] **Step 1.2: Add 6 module-level model classes to the test file** - -Insert immediately after the `_ModelWithPrivateAttr` class (around line 177), before the -`# ── Module-level models for read-path and round-trip tests ──` comment: - -```python -class _LiteralStrModel(BaseModel): - method: Literal["a", "b"] - - -class _LiteralIntModel(BaseModel): - count: Literal[1, 2, 3] - - -class _LiteralNoneModel(BaseModel): - status: Literal["active", None] - - -class _LiteralNoneOnlyModel(BaseModel): - x: Literal[None] - - -class _MixedLiteralModel(BaseModel): - val: Literal["a", 1] # type: ignore[assignment] - - -class _LiteralRoundTripModel(BaseModel): - method: Literal["a", "b"] - count: int -``` - -- [ ] **Step 1.3: Write the 5 failing registration-path tests** - -Append to the end of `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -# ── typing.Literal support tests (ITL-442) ─────────────────────────────────── - - -def test_factory_create_model_with_literal_str_field(): - """Literal["a", "b"] field → large_string in the Arrow struct.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_LiteralStrModel, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - assert storage.field("method").type == pa.large_string() - - -def test_factory_create_model_with_literal_int_field(): - """Literal[1, 2, 3] field → int64 in the Arrow struct.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_LiteralIntModel, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - assert storage.field("count").type == pa.int64() - - -def test_factory_create_model_with_literal_none_field(): - """Literal["active", None] strips None → resolves to large_string.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_LiteralNoneModel, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - assert storage.field("status").type == pa.large_string() - - -def test_factory_rejects_literal_none_only(): - """Literal[None] has no concrete value type — raises ValueError.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - with pytest.raises(ValueError, match="Literal\\[None\\]"): - factory.create_for_python_type(_LiteralNoneOnlyModel, converter=converter) - - -def test_factory_rejects_mixed_literal(): - """Literal["a", 1] mixes str and int — raises ValueError.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - with pytest.raises(ValueError, match="Mixed-type Literal"): - factory.create_for_python_type(_MixedLiteralModel, converter=converter) -``` - -- [ ] **Step 1.4: Run the 5 tests and confirm they all FAIL** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - -k "literal" -v 2>&1 | tail -20 -``` - -Expected: all 5 tests FAIL with `ValueError: Unsupported annotation: typing.Literal[...]` - -- [ ] **Step 1.5: Add the `Literal` branch to `_register_python_class_impl`** - -In `src/orcapod/semantic_types/universal_converter.py`, find `_register_python_class_impl` -(line 268). Locate the `Union/Optional` block — it looks like this (ends around line 295): - -```python - # Optional[T] / T | None → strip None arm - if origin is typing.Union or origin is _types_mod.UnionType: - non_none = [a for a in args if a is not type(None)] - if len(non_none) == 1: - return self.register_python_class(non_none[0]) - raise ValueError( - f"Complex unions with multiple non-None types are not supported: " - f"{annotation!r}. Only Optional[T] (T | None) is allowed." - ) - - # list[T] → pa.large_list(T). -``` - -Replace with (inserting the new `Literal` block between the `Union` block and the `list` comment): - -```python - # Optional[T] / T | None → strip None arm - if origin is typing.Union or origin is _types_mod.UnionType: - non_none = [a for a in args if a is not type(None)] - if len(non_none) == 1: - return self.register_python_class(non_none[0]) - raise ValueError( - f"Complex unions with multiple non-None types are not supported: " - f"{annotation!r}. Only Optional[T] (T | None) is allowed." - ) - - # typing.Literal[v1, v2, ...] → Arrow type of the literal values' type. - # None members are stripped (treat as optional/nullable); mixed non-None types raise. - if origin is typing.Literal: - value_types = {type(a) for a in args if a is not None} - if not value_types: - raise ValueError( - f"Literal[None] is not supported as an Arrow type. " - f"Use Optional[T] to express nullability instead." - ) - if len(value_types) != 1: - raise ValueError( - f"Mixed-type Literal is not supported: {annotation!r}. " - f"All members must share one type (e.g. Literal['a', 'b'])." - ) - return self.register_python_class(next(iter(value_types))) - - # list[T] → pa.large_list(T). -``` - -- [ ] **Step 1.6: Run the 5 tests and confirm they all PASS** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - -k "literal" -v 2>&1 | tail -20 -``` - -Expected: all 5 tests PASS. - -- [ ] **Step 1.7: Run the full pydantic factory test suite — confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v 2>&1 | tail -20 -``` - -Expected: all tests PASS. - -- [ ] **Step 1.8: Commit** - -```bash -git add tests/test_extension_types/test_pydantic_logical_type_factory.py \ - src/orcapod/semantic_types/universal_converter.py -git commit -m "fix(types): support typing.Literal in _register_python_class_impl (ITL-442)" -``` - ---- - -## Task 2: Serialization-path test + fix `_convert_python_to_arrow` - -**Files:** -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` -- Modify: `src/orcapod/semantic_types/universal_converter.py` - -**Why a second fix is needed:** `_create_python_to_arrow_converter` (the value-serialization path) calls `python_type_to_arrow_type` → `_convert_python_to_arrow` internally. Without this fix, value serialization of `Literal` fields inside pydantic models raises `ValueError: Unsupported generic type: typing.Literal` even though type registration now succeeds. - -- [ ] **Step 2.1: Write the failing round-trip test** - -Append to `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -def test_literal_model_round_trip(): - """python_to_storage → storage_to_python round-trip for a model with Literal fields.""" - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_LiteralRoundTripModel, converter=converter) - converter.register_logical_type(lt) - - instance = _LiteralRoundTripModel(method="a", count=42) - storage_value = lt.python_to_storage(instance, converter) - assert storage_value == {"method": "a", "count": 42} - - reconstructed = lt.storage_to_python(storage_value, converter) - assert isinstance(reconstructed, _LiteralRoundTripModel) - assert reconstructed.method == "a" - assert reconstructed.count == 42 -``` - -- [ ] **Step 2.2: Run the round-trip test and confirm it FAILS** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - ::test_literal_model_round_trip -v 2>&1 | tail -20 -``` - -Expected: FAIL with `ValueError: Unsupported generic type: typing.Literal` - -- [ ] **Step 2.3: Add the `Literal` branch to `_convert_python_to_arrow`** - -In `src/orcapod/semantic_types/universal_converter.py`, find `_convert_python_to_arrow` -(line ~997). Locate the `Union/Optional` elif block — it ends around line 1075 and is -immediately followed by `# Handle set types → lists`: - -```python - # Handle Union/Optional types - elif origin is typing.Union or origin is types.UnionType: - non_none_types = [t for t in args if t is not type(None)] - if len(non_none_types) == 1: - # Optional[T] → just T (nullability handled at field level) - return self.python_type_to_arrow_type(non_none_types[0]) - else: - raise ValueError( - f"Complex unions with multiple non-None types are not supported: {python_type}. " - f"Only Optional[T] (i.e., T | None) is allowed." - ) - - # Handle set types → lists - elif origin is set: -``` - -Replace with (inserting the `Literal` elif between `Union` and `set`): - -```python - # Handle Union/Optional types - elif origin is typing.Union or origin is types.UnionType: - non_none_types = [t for t in args if t is not type(None)] - if len(non_none_types) == 1: - # Optional[T] → just T (nullability handled at field level) - return self.python_type_to_arrow_type(non_none_types[0]) - else: - raise ValueError( - f"Complex unions with multiple non-None types are not supported: {python_type}. " - f"Only Optional[T] (i.e., T | None) is allowed." - ) - - # typing.Literal[v1, v2, ...] → Arrow type of the literal values' type. - # None members are stripped; mixed non-None types raise. - elif origin is typing.Literal: - value_types = {type(a) for a in args if a is not None} - if not value_types: - raise ValueError( - f"Literal[None] is not supported as an Arrow type. " - f"Use Optional[T] to express nullability instead." - ) - if len(value_types) != 1: - raise ValueError( - f"Mixed-type Literal is not supported: {python_type!r}. " - f"All members must share one type (e.g. Literal['a', 'b'])." - ) - return self.python_type_to_arrow_type(next(iter(value_types))) - - # Handle set types → lists - elif origin is set: -``` - -- [ ] **Step 2.4: Run the round-trip test and confirm it PASSES** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - ::test_literal_model_round_trip -v 2>&1 | tail -20 -``` - -Expected: PASS. - -- [ ] **Step 2.5: Run the full pydantic factory test suite — confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v 2>&1 | tail -20 -``` - -Expected: all tests PASS. - -- [ ] **Step 2.6: Commit** - -```bash -git add tests/test_extension_types/test_pydantic_logical_type_factory.py \ - src/orcapod/semantic_types/universal_converter.py -git commit -m "fix(types): support typing.Literal in _convert_python_to_arrow (ITL-442)" -``` - ---- - -## Task 3: End-to-end Arrow table conversion test - -**Files:** -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` - -This test exercises both fixes together via the full `register_python_class` → -`python_dicts_to_arrow_table` → `arrow_table_to_python_dicts` path, mirroring exactly what -`DictSource` does internally. - -- [ ] **Step 3.1: Write the end-to-end test** - -Append to `tests/test_extension_types/test_pydantic_logical_type_factory.py`: - -```python -def test_literal_model_as_dictsource_column(): - """Full Arrow table round-trip for a model with a Literal field (ITL-442 repro). - - Exercises the complete path: register_python_class → python_dicts_to_arrow_table → - arrow_table_to_python_dicts. This is the same sequence DictSource executes internally. - Before the fix, register_python_class raised: - ValueError: Unsupported annotation: typing.Literal['a', 'b'] - """ - converter = _make_full_converter() - - # Step A: register the model — previously raised ValueError - converter.register_python_class(_LiteralStrModel) - - # Step B: convert Python dicts to Arrow table (mirrors DictSource.__init__) - rows = [{"config": _LiteralStrModel(method="a")}] - arrow_schema = converter.python_schema_to_arrow_schema({"config": _LiteralStrModel}) - table = converter.python_dicts_to_arrow_table(rows, arrow_schema=arrow_schema) - - assert table.num_rows == 1 - assert "config" in table.schema.names - - # Step C: round-trip back to Python dicts - result = converter.arrow_table_to_python_dicts(table) - assert len(result) == 1 - assert isinstance(result[0]["config"], _LiteralStrModel) - assert result[0]["config"].method == "a" -``` - -- [ ] **Step 3.2: Run the end-to-end test and confirm it PASSES** - -```bash -uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py \ - ::test_literal_model_as_dictsource_column -v 2>&1 | tail -20 -``` - -Expected: PASS. - -- [ ] **Step 3.3: Run the full extension-types test suite** - -```bash -uv run pytest tests/test_extension_types/ -v 2>&1 | tail -30 -``` - -Expected: all tests PASS. - -- [ ] **Step 3.4: Run the full test suite** - -```bash -uv run pytest tests/ -x --ignore=tests/test_channels --ignore=tests/test_databases \ - -q 2>&1 | tail -30 -``` - -(`--ignore` skips integration tests that require external services. `-x` stops on first -failure so errors are easy to read.) - -Expected: all tests PASS. - -- [ ] **Step 3.5: Commit** - -```bash -git add tests/test_extension_types/test_pydantic_logical_type_factory.py -git commit -m "test(types): add end-to-end Literal model Arrow round-trip test (ITL-442)" -``` - ---- - -## Self-Review - -**Spec coverage:** -- `_register_python_class_impl` fix → Task 1, Step 1.5 ✓ -- `_convert_python_to_arrow` fix → Task 2, Step 2.3 ✓ -- `Literal["a", "b"]` → `large_string` → Task 1, test 1 ✓ -- `Literal[1, 2]` → `int64` → Task 1, test 2 ✓ -- `Literal["a", None]` → `large_string` (None stripped) → Task 1, test 3 ✓ -- `Literal[None]` raises `ValueError` → Task 1, test 4 ✓ -- Mixed `Literal["a", 1]` raises `ValueError` → Task 1, test 5 ✓ -- Round-trip value serialization → Task 2, test 6 ✓ -- End-to-end DictSource path → Task 3, test 7 ✓ - -**Type consistency:** `_LiteralStrModel`, `_LiteralRoundTripModel`, etc. are defined once in -Step 1.2 and referenced identically in all subsequent tests. ✓ - -**No placeholders.** ✓ diff --git a/superpowers/plans/2026-06-30-itl-453-union-signature-hash-order-independence.md b/superpowers/plans/2026-06-30-itl-453-union-signature-hash-order-independence.md deleted file mode 100644 index 427f5c0ef..000000000 --- a/superpowers/plans/2026-06-30-itl-453-union-signature-hash-order-independence.md +++ /dev/null @@ -1,599 +0,0 @@ -# Union Signature Hash Order Independence Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `_function_signature_hash` order-independent for union-typed parameters and return types so that `def foo(x: str | Path)` and `def foo(x: Path | str)` produce the same hash. - -**Architecture:** Add two private helpers (`_is_union_annotation`, `_canonical_annotation_str`) to `hash_utils.py` that canonicalize union type annotations by sorting their members lexicographically. Apply them at two call sites: `get_function_signature()` (which produces `_function_signature_hash`) and `FunctionSignatureExtractor.extract_function_info()` (which handles function objects passed directly to `hash_object()`). Non-union annotations are unchanged byte-for-byte. - -**Tech Stack:** Python 3.12, `inspect` stdlib module, `types` stdlib module, `typing` stdlib module, uv (test runner: `uv run pytest`) - ---- - -## File Map - -| File | Action | Purpose | -|---|---|---| -| `src/orcapod/hashing/hash_utils.py` | Modify | Add `_is_union_annotation`, `_canonical_annotation_str`; canonicalize unions in `get_function_signature` | -| `src/orcapod/hashing/semantic_hashing/function_info_extractors.py` | Modify | Canonicalize union param annotations in `FunctionSignatureExtractor.extract_function_info` | -| `tests/test_hashing/test_hash_utils.py` | Create | Unit tests for `get_function_signature` and `_canonical_annotation_str` | -| `tests/test_core/data_function/test_data_function.py` | Modify | Add `TestSignatureHashUnionOrderIndependence` integration tests | -| `DESIGN_ISSUES.md` | Modify | Add resolved entry for this bug | - ---- - -### Task 1: Check out the feature branch - -**Files:** none (git only) - -- [ ] **Step 1: Check out the branch** - -```bash -git checkout -b eywalker/itl-453-function-signature-hash-must-be-order-independent-for-union -``` - -- [ ] **Step 2: Verify** - -```bash -git branch --show-current -``` - -Expected output: `eywalker/itl-453-function-signature-hash-must-be-order-independent-for-union` - ---- - -### Task 2: Write failing tests for `get_function_signature` canonicalization - -**Files:** -- Create: `tests/test_hashing/test_hash_utils.py` - -These tests will fail until the helpers are added in Task 3. - -- [ ] **Step 1: Create `tests/test_hashing/test_hash_utils.py`** - -```python -"""Tests for hash_utils helpers, specifically canonical union annotation strings.""" -import inspect -from pathlib import Path - -import pytest - -from orcapod.hashing.hash_utils import ( - _canonical_annotation_str, - _is_union_annotation, - get_function_signature, -) - - -class TestIsUnionAnnotation: - def test_pep604_union_detected(self): - assert _is_union_annotation(str | Path) is True - - def test_pep604_three_member_union_detected(self): - assert _is_union_annotation(str | Path | bytes) is True - - def test_typing_union_detected(self): - import typing - assert _is_union_annotation(typing.Union[str, int]) is True - - def test_typing_optional_detected(self): - import typing - assert _is_union_annotation(typing.Optional[str]) is True - - def test_plain_type_not_union(self): - assert _is_union_annotation(int) is False - - def test_generic_alias_not_union(self): - assert _is_union_annotation(list[str]) is False - - def test_none_type_not_union(self): - assert _is_union_annotation(type(None)) is False - - -class TestCanonicalAnnotationStr: - def test_pep604_two_member_order_independent(self): - """str | Path and Path | str produce the same canonical string.""" - assert _canonical_annotation_str(str | Path) == _canonical_annotation_str(Path | str) - - def test_pep604_canonical_form(self): - """str | Path canonicalizes to 'pathlib.Path | str' (P before s).""" - assert _canonical_annotation_str(str | Path) == "pathlib.Path | str" - - def test_pep604_three_member_order_independent(self): - """All permutations of str | Path | bytes produce the same canonical string.""" - canonical = _canonical_annotation_str(str | Path | bytes) - assert _canonical_annotation_str(bytes | str | Path) == canonical - assert _canonical_annotation_str(Path | bytes | str) == canonical - - def test_pep604_three_member_canonical_form(self): - """str | Path | bytes canonicalizes to 'bytes | pathlib.Path | str'.""" - assert _canonical_annotation_str(str | Path | bytes) == "bytes | pathlib.Path | str" - - def test_non_union_matches_formatannotation(self): - """Non-union types fall through to inspect.formatannotation.""" - for t in (int, str, bytes, Path): - assert _canonical_annotation_str(t) == inspect.formatannotation(t) - - def test_typing_union_order_independent(self): - import typing - assert ( - _canonical_annotation_str(typing.Union[str, int]) - == _canonical_annotation_str(typing.Union[int, str]) - ) - - -class TestGetFunctionSignatureUnionCanonical: - def test_param_union_order_independent(self): - """get_function_signature returns the same string for str|Path and Path|str params.""" - def foo1(x: str | Path) -> str: - return str(x) - - def foo2(x: Path | str) -> str: - return str(x) - - assert get_function_signature(foo1) == get_function_signature(foo2) - - def test_param_three_member_union_order_independent(self): - """All permutations of a 3-member union param produce the same signature string.""" - def f1(x: str | Path | bytes) -> str: - return str(x) - - def f2(x: bytes | str | Path) -> str: - return str(x) - - def f3(x: Path | bytes | str) -> str: - return str(x) - - sig1 = get_function_signature(f1) - assert get_function_signature(f2) == sig1 - assert get_function_signature(f3) == sig1 - - def test_return_union_order_independent(self): - """get_function_signature returns the same string for str|Path and Path|str returns.""" - def foo1(x: int) -> str | Path: - return str(x) - - def foo2(x: int) -> Path | str: - return str(x) - - assert get_function_signature(foo1) == get_function_signature(foo2) - - def test_non_union_param_unchanged(self): - """Non-union param signatures are byte-for-byte identical before and after.""" - def foo(x: int) -> str: - return str(x) - - # The exact current format — verifies no regression for non-union types. - # Note: return type uses str(annotation) = "" (not "str"). - sig = get_function_signature(foo) - assert "x: int" in sig - assert "foo" in sig - - def test_non_union_signature_stable(self): - """Non-union function signature is deterministic across calls.""" - def bar(a: int, b: str) -> bytes: - return b.encode() - - assert get_function_signature(bar) == get_function_signature(bar) -``` - -- [ ] **Step 2: Run to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_hash_utils.py -v 2>&1 | head -40 -``` - -Expected: `ImportError` or `FAILED` because `_canonical_annotation_str` and `_is_union_annotation` don't exist yet. - ---- - -### Task 3: Add helpers and fix `get_function_signature` in `hash_utils.py` - -**Files:** -- Modify: `src/orcapod/hashing/hash_utils.py` - -The file currently lives at `src/orcapod/hashing/hash_utils.py`. The relevant function is `get_function_signature` at lines 142–188. We add two private helpers above it and modify the function body. - -- [ ] **Step 1: Add `_is_union_annotation` helper after the imports block (after line 13)** - -Insert after the `logger = logging.getLogger(__name__)` line: - -```python -def _is_union_annotation(annotation: object) -> bool: - """Return ``True`` if *annotation* is a union type. - - Detects both PEP 604 ``X | Y`` (``types.UnionType``) and - ``typing.Union[X, Y]`` / ``typing.Optional[X]``. - - Args: - annotation: Any Python object (type annotation or otherwise). - - Returns: - ``True`` if the annotation is a union; ``False`` otherwise. - """ - import types as _types - import typing - - if isinstance(annotation, _types.UnionType): - return True - return getattr(annotation, "__origin__", None) is typing.Union - - -def _canonical_annotation_str(annotation: object) -> str: - """Return a stable, canonical string for a type annotation. - - For union types (both PEP 604 ``X | Y`` and ``typing.Union[X, Y]``), - members are sorted byte-wise so that ``str | Path`` and ``Path | str`` - produce the same canonical string. Non-union types fall through to - ``inspect.formatannotation``, preserving existing behaviour exactly. - - The canonical ordering key is the fully qualified type name produced by - ``inspect.formatannotation`` (e.g. ``"pathlib.Path"``, ``"str"``), - sorted lexicographically. This is stable across Python versions and - machines and does not depend on ``id()`` or ``__hash__``. - - Args: - annotation: A type annotation object. - - Returns: - A canonical string representation. - """ - if _is_union_annotation(annotation): - args = getattr(annotation, "__args__", ()) or () - member_strs = sorted(_canonical_annotation_str(a) for a in args) - return " | ".join(member_strs) - return inspect.formatannotation(annotation) -``` - -- [ ] **Step 2: Modify `get_function_signature` to canonicalize union params** - -In `get_function_signature`, replace the parameter loop (lines 170–175): - -```python - param_strs = [] - for name, param in sig.parameters.items(): - param_str = str(param) - if not include_defaults and "=" in param_str: - param_str = param_str.split("=")[0].strip() - param_strs.append(param_str) -``` - -with: - -```python - param_strs = [] - for name, param in sig.parameters.items(): - param_str = str(param) - annotation = param.annotation - if annotation is not inspect.Parameter.empty and _is_union_annotation(annotation): - old_ann = inspect.formatannotation(annotation) - new_ann = _canonical_annotation_str(annotation) - # Replace ": " with ": " (first occurrence only). - # The ": " prefix distinguishes the annotation from the default value. - param_str = param_str.replace(f": {old_ann}", f": {new_ann}", 1) - if not include_defaults and "=" in param_str: - param_str = param_str.split("=")[0].strip() - param_strs.append(param_str) -``` - -- [ ] **Step 3: Modify `get_function_signature` to canonicalize union return types** - -Replace the return-type line (lines 186–187): - -```python - if "returns" in parts: - fn_string += f"-> {parts['returns']}" -``` - -with: - -```python - if "returns" in parts: - ret = parts["returns"] - if _is_union_annotation(ret): - fn_string += f"-> {_canonical_annotation_str(ret)}" - else: - fn_string += f"-> {ret}" -``` - -- [ ] **Step 4: Run the new hash_utils tests** - -```bash -uv run pytest tests/test_hashing/test_hash_utils.py -v -``` - -Expected: all tests **PASS**. - -- [ ] **Step 5: Run existing hashing tests to confirm no regression** - -```bash -uv run pytest tests/test_hashing/ -v 2>&1 | tail -20 -``` - -Expected: all existing tests still **PASS**. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/hash_utils.py tests/test_hashing/test_hash_utils.py -git commit -m "fix(hashing): canonicalize union annotation order in get_function_signature (ITL-453)" -``` - ---- - -### Task 4: Fix `FunctionSignatureExtractor` for consistency - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/function_info_extractors.py` - -`FunctionSignatureExtractor.extract_function_info` builds a `params` string with the same `str(param)` pattern. Its `returns` field is already a type object (order-independent via `UnionTypeHandler`), so only `params` needs fixing. - -- [ ] **Step 1: Import the helpers at the top of `function_info_extractors.py`** - -Add to the existing imports block: - -```python -from orcapod.hashing.hash_utils import _canonical_annotation_str, _is_union_annotation -``` - -- [ ] **Step 2: Apply the same param-string substitution in `extract_function_info`** - -In `FunctionSignatureExtractor.extract_function_info`, replace the parameter loop (lines 61–67): - -```python - param_strs = [] - for name, param in sig.parameters.items(): - param_str = str(param) - if not self.include_defaults and "=" in param_str: - param_str = param_str.split("=")[0].strip() - - param_strs.append(param_str) -``` - -with: - -```python - param_strs = [] - for name, param in sig.parameters.items(): - param_str = str(param) - annotation = param.annotation - if annotation is not inspect.Parameter.empty and _is_union_annotation(annotation): - old_ann = inspect.formatannotation(annotation) - new_ann = _canonical_annotation_str(annotation) - param_str = param_str.replace(f": {old_ann}", f": {new_ann}", 1) - if not self.include_defaults and "=" in param_str: - param_str = param_str.split("=")[0].strip() - param_strs.append(param_str) -``` - -- [ ] **Step 3: Run existing tests** - -```bash -uv run pytest tests/test_hashing/ tests/test_core/data_function/ -v 2>&1 | tail -20 -``` - -Expected: all tests **PASS**. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/function_info_extractors.py -git commit -m "fix(hashing): canonicalize union annotation order in FunctionSignatureExtractor (ITL-453)" -``` - ---- - -### Task 5: Add integration tests to `test_data_function.py` - -**Files:** -- Modify: `tests/test_core/data_function/test_data_function.py` - -These tests verify the end-to-end property: `PythonDataFunction._function_signature_hash` is order-independent for union-typed parameters and return types. - -- [ ] **Step 1: Add the import for Path at the top of the test file if not present** - -Check whether `from pathlib import Path` is already imported. If not, add it with the other imports near the top of the file. - -- [ ] **Step 2: Append the new test class at the end of the file** - -```python -# --------------------------------------------------------------------------- -# TestSignatureHashUnionOrderIndependence -# --------------------------------------------------------------------------- - - -class TestSignatureHashUnionOrderIndependence: - """_function_signature_hash must be order-independent over union members.""" - - def _sig_hash(self, func): - # PythonDataFunction is already imported at the top of this test file. - df = PythonDataFunction(func, output_keys="result") - return df.get_function_variation_data()["function_signature_hash"] - - def test_two_member_union_param_order_independent(self): - """str | Path and Path | str produce the same signature hash.""" - def foo1(x: str | Path) -> str: - return str(x) - - def foo2(x: Path | str) -> str: - return str(x) - - assert self._sig_hash(foo1) == self._sig_hash(foo2) - - def test_three_member_union_all_permutations(self): - """All permutations of str | Path | bytes produce the same signature hash.""" - def f1(x: str | Path | bytes) -> str: - return str(x) - - def f2(x: bytes | str | Path) -> str: - return str(x) - - def f3(x: Path | bytes | str) -> str: - return str(x) - - h1 = self._sig_hash(f1) - assert self._sig_hash(f2) == h1 - assert self._sig_hash(f3) == h1 - - def test_return_type_union_order_independent(self): - """Return-type unions are also order-independent.""" - def foo1(x: int) -> str | Path: - return str(x) - - def foo2(x: int) -> Path | str: - return str(x) - - assert self._sig_hash(foo1) == self._sig_hash(foo2) - - def test_non_union_param_hash_unchanged(self): - """Non-union functions produce the same hash as a second identical definition.""" - def foo(x: int) -> str: - return str(x) - - def foo_copy(x: int) -> str: - return str(x) - - # Both definitions are structurally identical (non-union) — hashes must agree. - # This exercises that the fix does not disturb non-union annotations. - assert self._sig_hash(foo) == self._sig_hash(foo_copy) - - def test_canonical_union_ordering(self): - """str | Path canonicalizes to 'pathlib.Path | str' (P before s).""" - from orcapod.hashing.hash_utils import _canonical_annotation_str - assert _canonical_annotation_str(str | Path) == "pathlib.Path | str" - - def test_different_union_types_still_differ(self): - """str | Path and str | bytes are different and must not hash the same.""" - def foo1(x: str | Path) -> str: - return str(x) - - def foo2(x: str | bytes) -> str: - return str(x) - - assert self._sig_hash(foo1) != self._sig_hash(foo2) - - def test_union_vs_non_union_differ(self): - """A union-typed param and a plain-typed param produce different hashes.""" - def foo1(x: str | Path) -> str: - return str(x) - - def foo2(x: str) -> str: - return str(x) - - assert self._sig_hash(foo1) != self._sig_hash(foo2) -``` - -- [ ] **Step 3: Run the new integration tests** - -```bash -uv run pytest tests/test_core/data_function/test_data_function.py::TestSignatureHashUnionOrderIndependence -v -``` - -Expected: all 7 tests **PASS**. - -- [ ] **Step 4: Run the full data_function test suite** - -```bash -uv run pytest tests/test_core/data_function/ -v 2>&1 | tail -20 -``` - -Expected: all tests **PASS**. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_core/data_function/test_data_function.py -git commit -m "test(data_function): add union signature hash order-independence tests (ITL-453)" -``` - ---- - -### Task 6: Update `DESIGN_ISSUES.md` - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -Per project convention, add a resolved entry for this bug. Find the section containing the UC (Union / Converter) issues (around the `### UC2` entry near line 1161) and add a new entry after UC2. - -- [ ] **Step 1: Add the UC3 entry after the UC2 block** - -Insert after the `---` that follows the UC2 block: - -```markdown -### UC3 — Function signature hash order-dependent for union-typed annotations -**Status:** resolved -**Severity:** high -**Issue:** ITL-453 - -``get_function_signature()`` in ``hash_utils.py`` used ``str(param)`` to build -the signature string. For union type annotations, ``str(param)`` calls -``inspect.formatannotation``, which falls through to ``repr(annotation)`` — -reflecting declaration order. Two semantically identical signatures like -``foo(x: str | Path)`` and ``foo(x: Path | str)`` therefore produced different -``_function_signature_hash`` values, silently breaking content addressability -for any pipeline that refactored union member order. - -**Fix:** Added ``_is_union_annotation`` and ``_canonical_annotation_str`` -helpers to ``hash_utils.py``. Union members are now sorted byte-wise by their -``inspect.formatannotation`` string before being embedded in the signature. -Applied the same fix to ``FunctionSignatureExtractor.extract_function_info``. -Non-union annotations are byte-for-byte unchanged. - ---- -``` - -- [ ] **Step 2: Commit** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs(design-issues): add UC3 resolved entry for ITL-453 union signature hash" -``` - ---- - -### Task 7: Full test suite and push - -- [ ] **Step 1: Run the full test suite** - -```bash -uv run pytest tests/ -x --timeout=120 2>&1 | tail -30 -``` - -Expected: all tests **PASS** (no failures, no errors). - -- [ ] **Step 2: Push the branch** - -```bash -git push -u origin eywalker/itl-453-function-signature-hash-must-be-order-independent-for-union -``` - -- [ ] **Step 3: Create the PR** - -```bash -gh pr create \ - --title "fix(hashing): make union-typed signature hash order-independent (ITL-453)" \ - --body "$(cat <<'EOF' -## Summary - -- Adds `_is_union_annotation` and `_canonical_annotation_str` helpers to `hash_utils.py` -- Fixes `get_function_signature` to sort union members before embedding them in the signature string — both for parameter annotations and return type annotations -- Fixes `FunctionSignatureExtractor.extract_function_info` for the same bug in its `params` field -- Non-union annotations are byte-for-byte unchanged (no regression to existing hashes) - -## Breaking change - -Functions whose parameter or return type annotations include a union type (`str | Path`, `typing.Union[X, Y]`, `typing.Optional[X]`) will receive a different `_function_signature_hash` after this fix. Cached pipeline outputs keyed on such hashes will be invalidated. Acceptable under the v0.1 green-field stance. - -Fixes ITL-453 - -## Test plan - -- [ ] `uv run pytest tests/test_hashing/test_hash_utils.py -v` — new unit tests for helpers and `get_function_signature` -- [ ] `uv run pytest tests/test_core/data_function/test_data_function.py::TestSignatureHashUnionOrderIndependence -v` — integration tests for `_function_signature_hash` -- [ ] `uv run pytest tests/ -x` — full suite passes with no regressions - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" \ - --base main -``` diff --git a/superpowers/plans/2026-06-30-op-file-type.md b/superpowers/plans/2026-06-30-op-file-type.md deleted file mode 100644 index e30ebb2e8..000000000 --- a/superpowers/plans/2026-06-30-op-file-type.md +++ /dev/null @@ -1,1046 +0,0 @@ -# op.File Type and pathlib.Path Pure-Path Refactor — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Introduce `orcapod.File` as a content-identified file type backed by `ProxyUPath`, and remove content-hashing from `pathlib.Path` and `upath.UPath` so they hash from their path string. - -**Architecture:** `File` subclasses `upath.extensions.ProxyUPath` and validates existence on construction. A new `LogicalFile` Arrow extension type serialises `File` as a path string (`"orcapod.file"`). A new `FileHandler` replaces the removed `PathHandler` and `UPathHandler` in the semantic hasher registry. - -**Tech Stack:** Python 3.12, `upath >= 0.3.8`, `pyarrow >= 20`, `polars >= 1.36`, `uv run pytest` for tests. - -**Spec:** `superpowers/specs/2026-06-29-op-file-type-design.md` - ---- - -## File Map - -| Action | Path | Responsibility | -|---|---|---| -| **Create** | `src/orcapod/extension_types/file_type.py` | `File` class + `LogicalFile` Arrow extension | -| **Modify** | `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Add `FileHandler`; delete `PathHandler`, `UPathHandler` | -| **Modify** | `src/orcapod/hashing/semantic_hashing/__init__.py` | Swap `PathHandler`/`UPathHandler` for `FileHandler` in exports | -| **Modify** | `src/orcapod/extension_types/__init__.py` | Add `LogicalFile` to exports | -| **Modify** | `src/orcapod/contexts/data/v0.1.json` | Add `LogicalFile`; remove Path/UPath handlers; add `File` handler | -| **Modify** | `src/orcapod/__init__.py` | Add `orcapod.File` stable export | -| **Create** | `tests/test_extension_types/test_file_type.py` | Tests for `File` and `LogicalFile` | -| **Create** | `tests/test_hashing/test_file_handler.py` | Tests for `FileHandler` | -| **Modify** | `tests/test_hashing/test_extension_type_hashing.py` | Port Path tests → File; add no-read test for Path | -| **Modify** | `tests/test_hashing/test_semantic_hasher.py` | Remove `PathHandler`-dependent tests | -| **Modify** | `CHANGELOG.md` | Breaking-change entry | - ---- - -## Task 1: Create branch - -**Files:** none (git only) - -- [ ] **Step 1: Check out the feature branch** - -```bash -git checkout -b eywalker/itl-450-add-opfile-type-and-refactor-pathlibpath-to-be-a-pure-path -git branch --show-current -``` - -Expected output: `eywalker/itl-450-add-opfile-type-and-refactor-pathlibpath-to-be-a-pure-path` - ---- - -## Task 2: `File` class (existence-validated ProxyUPath) - -**Files:** -- Create: `src/orcapod/extension_types/file_type.py` -- Create: `tests/test_extension_types/test_file_type.py` - -- [ ] **Step 1: Write the failing tests for `File`** - -Create `tests/test_extension_types/test_file_type.py`: - -```python -"""Tests for orcapod.extension_types.file_type.File.""" - -from __future__ import annotations - -import os -import pytest - -from orcapod.extension_types.file_type import File - - -class TestFileConstructor: - def test_rejects_nonexistent_path(self, tmp_path): - with pytest.raises(FileNotFoundError): - File(tmp_path / "does_not_exist.txt") - - def test_rejects_directory(self, tmp_path): - with pytest.raises(IsADirectoryError): - File(tmp_path) - - def test_rejects_symlink_to_directory_when_following(self, tmp_path): - target_dir = tmp_path / "real_dir" - target_dir.mkdir() - link = tmp_path / "link_to_dir" - link.symlink_to(target_dir) - with pytest.raises(IsADirectoryError): - File(link) # follow_symlinks=True default, target is a dir - - def test_rejects_symlink_when_follow_symlinks_false(self, tmp_path): - real_file = tmp_path / "real.txt" - real_file.write_text("content") - link = tmp_path / "link_to_file" - link.symlink_to(real_file) - with pytest.raises(ValueError, match="symlink"): - File(link, follow_symlinks=False) - - def test_accepts_symlink_to_file_when_following(self, tmp_path): - real_file = tmp_path / "real.txt" - real_file.write_text("content") - link = tmp_path / "link_to_file" - link.symlink_to(real_file) - f = File(link) # follow_symlinks=True default - assert str(f) == str(link) - - def test_accepts_zero_byte_file(self, tmp_path): - empty = tmp_path / "empty.txt" - empty.write_bytes(b"") - f = File(empty) - assert str(f) == str(empty) - - def test_accepts_regular_file(self, tmp_path): - regular = tmp_path / "regular.txt" - regular.write_text("hello") - f = File(regular) - assert str(f) == str(regular) - - def test_str_returns_path_string(self, tmp_path): - p = tmp_path / "file.txt" - p.write_text("data") - f = File(p) - assert str(f) == str(p) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py -v -``` - -Expected: `ModuleNotFoundError` or `ImportError` — `file_type` does not exist yet. - -- [ ] **Step 3: Implement `File` in `src/orcapod/extension_types/file_type.py`** - -Create `src/orcapod/extension_types/file_type.py`: - -```python -"""orcapod.File — content-identified, existence-validated file path. - -``File`` wraps a ``upath.UPath`` and validates that the path points to a readable, -non-directory file at construction time. Use ``pathlib.Path`` / ``upath.UPath`` for -paths that may not yet exist. - -``LogicalFile`` is the Arrow extension type that serialises ``File`` instances as -``large_string`` columns tagged with the ``"orcapod.file"`` extension name. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -import polars as pl -import pyarrow as pa -from upath import UPath -from upath.extensions import ProxyUPath - -from orcapod.extension_types.base_logical_type import BaseLogicalType -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type - -if TYPE_CHECKING: - from orcapod.extension_types.protocols import TypeConverterProtocol - - -class File(ProxyUPath): - """A content-identified, existence-validated file path. - - Wraps a ``UPath`` and validates that the path refers to a readable, non-directory - file at construction time. Works across local, S3, GCS, and any other - fsspec-backed backend supported by ``upath``. - - Note: - ``isinstance(file_instance, UPath)`` returns ``False`` because ``ProxyUPath`` - does not inherit from ``UPath``. Use ``isinstance(x, File)`` to type-check. - - Args: - *args: Positional path arguments forwarded to ``UPath``. - follow_symlinks: If ``True`` (default), symlinks are followed and the - resolved target is validated and hashed. If ``False``, raises - ``ValueError`` on construction when the path is a symlink. - **kwargs: Keyword arguments forwarded to ``UPath``. - - Raises: - FileNotFoundError: If the path does not exist. - IsADirectoryError: If the path is a directory (or a symlink to one when - ``follow_symlinks=True``). - ValueError: If the path is a symlink and ``follow_symlinks=False``, or if - the path exists but is not a regular file. - - Example: - >>> f = File("/tmp/data.csv") - >>> str(f) - '/tmp/data.csv' - >>> File("/tmp/nonexistent.csv") - FileNotFoundError: ... - """ - - def __init__(self, *args: Any, follow_symlinks: bool = True, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - if not follow_symlinks and self.__wrapped__.is_symlink(): - raise ValueError( - f"File: path is a symlink and follow_symlinks=False: {self.__wrapped__!r}" - ) - if not self.__wrapped__.exists(): - raise FileNotFoundError( - f"File: path does not exist: {self.__wrapped__!r}" - ) - if self.__wrapped__.is_dir(): - raise IsADirectoryError( - f"File: path is a directory: {self.__wrapped__!r}" - ) - if not self.__wrapped__.is_file(): - raise ValueError( - f"File: path is not a regular file: {self.__wrapped__!r}" - ) - - @classmethod - def _from_upath(cls, upath: UPath) -> "File": - """Create a ``File`` from an existing ``UPath`` without validation. - - Used internally by ``ProxyUPath`` for derived paths (e.g. ``.parent``, - ``/`` operator). Validation is intentionally skipped — derived paths from - navigation may not exist yet. ``follow_symlinks`` defaults to ``True`` on - all derived instances. - """ - obj = object.__new__(cls) - obj.__wrapped__ = upath - return obj -``` - -- [ ] **Step 4: Run the tests** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py -v -``` - -Expected: All 8 tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/file_type.py tests/test_extension_types/test_file_type.py -git commit -m "feat(types): add orcapod.File existence-validated ProxyUPath type (ITL-450)" -``` - ---- - -## Task 3: `LogicalFile` Arrow extension type - -**Files:** -- Modify: `src/orcapod/extension_types/file_type.py` -- Modify: `tests/test_extension_types/test_file_type.py` - -- [ ] **Step 1: Write the failing tests for `LogicalFile`** - -Append to `tests/test_extension_types/test_file_type.py`: - -```python -import pyarrow as pa -from orcapod.extension_types.file_type import File, LogicalFile - - -class TestLogicalFile: - def test_logical_type_name(self): - lt = LogicalFile() - assert lt.logical_type_name == "orcapod.file" - - def test_python_type(self): - lt = LogicalFile() - assert lt.python_type is File - - def test_arrow_ext_name(self): - lt = LogicalFile() - assert lt.get_arrow_extension_type().extension_name == "orcapod.file" - - def test_arrow_ext_storage_type(self): - lt = LogicalFile() - assert lt.get_arrow_extension_type().storage_type == pa.large_string() - - def test_python_to_storage_returns_string(self, tmp_path): - p = tmp_path / "f.txt" - p.write_text("x") - f = File(p) - lt = LogicalFile() - result = lt.python_to_storage(f) - assert result == str(p) - assert isinstance(result, str) - - def test_storage_to_python_returns_file(self, tmp_path): - p = tmp_path / "f.txt" - p.write_text("x") - lt = LogicalFile() - result = lt.storage_to_python(str(p)) - assert isinstance(result, File) - assert str(result) == str(p) - - def test_round_trip_preserves_path(self, tmp_path): - p = tmp_path / "f.txt" - p.write_text("round trip") - f = File(p) - lt = LogicalFile() - storage = lt.python_to_storage(f) - recovered = lt.storage_to_python(storage) - assert str(recovered) == str(f) - - def test_storage_to_python_raises_if_file_missing(self, tmp_path): - lt = LogicalFile() - with pytest.raises(FileNotFoundError): - lt.storage_to_python(str(tmp_path / "gone.txt")) - - def test_arrow_extension_type_is_cached(self): - lt = LogicalFile() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py::TestLogicalFile -v -``` - -Expected: `ImportError` — `LogicalFile` not defined yet. - -- [ ] **Step 3: Implement `LogicalFile` in `src/orcapod/extension_types/file_type.py`** - -Append to `file_type.py` (after the `File` class): - -```python -class LogicalFile(BaseLogicalType): - """Logical type for ``orcapod.File``. - - Stores ``File`` instances as Arrow large strings using the custom extension - type ``"orcapod.file"``. The stored value is the path string (e.g. - ``"/tmp/data.csv"`` or ``"s3://bucket/key"``). - - On read (``storage_to_python``), the path is used to reconstruct a ``File`` - instance, which re-validates existence. Reading an Arrow table with - ``"orcapod.file"`` columns will raise ``FileNotFoundError`` if the underlying - files have been moved or deleted — this is the correct semantic for a - content-identified type. - - Example: - >>> import tempfile, pathlib - >>> lt = LogicalFile() - >>> with tempfile.NamedTemporaryFile(delete=False) as f: - ... _ = f.write(b"hello") - ... tmp = f.name - >>> file = File(tmp) - >>> lt.storage_to_python(lt.python_to_storage(file)) == file - True - """ - - _arrow_ext_class = make_arrow_extension_type("orcapod.file", pa.large_string()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("orcapod.file", pa.large_string()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "orcapod.file" - python_type: type = File - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for ``File``. - - Returns: - A cached ``pa.ExtensionType`` with extension name ``"orcapod.file"`` - and storage type ``pa.large_string()``. - """ - if LogicalFile._arrow_ext is None: - LogicalFile._arrow_ext = LogicalFile._arrow_ext_class() - return LogicalFile._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for ``File``. - - Returns: - A cached ``pl.BaseExtension`` registered under ``"orcapod.file"``. - """ - if LogicalFile._polars_ext is None: - LogicalFile._polars_ext = LogicalFile._polars_ext_class() - return LogicalFile._polars_ext - - def python_to_storage(self, value: Any, converter: "TypeConverterProtocol | None" = None) -> str: - """Convert a ``File`` to its string path representation. - - Args: - value: A ``File`` instance. - converter: Ignored. Present for protocol conformance. - - Returns: - The string form of the path (e.g. ``"/tmp/data.csv"``). - """ - return str(value) - - def storage_to_python(self, storage_value: Any, converter: "TypeConverterProtocol | None" = None) -> File: - """Reconstruct a ``File`` from its stored string path. - - Re-validates existence on read — raises ``FileNotFoundError`` if the file - no longer exists at the stored path. - - Args: - storage_value: A string path as stored in Arrow. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``File`` instance. - - Raises: - FileNotFoundError: If the path no longer exists. - IsADirectoryError: If the path is now a directory. - """ - return File(storage_value) -``` - -- [ ] **Step 4: Run the tests** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py -v -``` - -Expected: All tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/file_type.py tests/test_extension_types/test_file_type.py -git commit -m "feat(types): add LogicalFile Arrow extension type for orcapod.File (ITL-450)" -``` - ---- - -## Task 4: `FileHandler` content hasher - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Create: `tests/test_hashing/test_file_handler.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_hashing/test_file_handler.py`: - -```python -"""Tests for FileHandler — content hashing for orcapod.File.""" - -from __future__ import annotations - -import hashlib -import pytest - -from orcapod.extension_types.file_type import File -from orcapod.hashing.file_hashers import BasicFileHasher -from orcapod.hashing.semantic_hashing.builtin_handlers import FileHandler -from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher -from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeHandlerRegistry -from orcapod.types import ContentHash - - -@pytest.fixture -def file_hasher(): - return BasicFileHasher(algorithm="sha256") - - -@pytest.fixture -def handler(file_hasher): - return FileHandler(file_hasher) - - -@pytest.fixture -def hasher(handler): - registry = PythonTypeHandlerRegistry() - registry.register(File, handler) - return SemanticAwarePythonHasher( - hasher_id="test_file_v0", - type_handler_registry=registry, - ) - - -class TestFileHandler: - def test_returns_content_hash(self, handler, hasher, tmp_path): - p = tmp_path / "a.txt" - p.write_text("hello") - f = File(p) - result = handler.handle(f, hasher) - assert isinstance(result, ContentHash) - - def test_same_content_same_hash(self, handler, hasher, tmp_path): - p1 = tmp_path / "a.txt" - p2 = tmp_path / "b.txt" - p1.write_bytes(b"identical") - p2.write_bytes(b"identical") - h1 = handler.handle(File(p1), hasher) - h2 = handler.handle(File(p2), hasher) - assert h1 == h2 - - def test_different_content_different_hash(self, handler, hasher, tmp_path): - p1 = tmp_path / "a.txt" - p2 = tmp_path / "b.txt" - p1.write_bytes(b"content A") - p2.write_bytes(b"content B") - h1 = handler.handle(File(p1), hasher) - h2 = handler.handle(File(p2), hasher) - assert h1 != h2 - - def test_zero_byte_file_produces_hash(self, handler, hasher, tmp_path): - p = tmp_path / "empty.txt" - p.write_bytes(b"") - f = File(p) - result = handler.handle(f, hasher) - assert isinstance(result, ContentHash) - - def test_zero_byte_file_hash_is_consistent(self, handler, hasher, tmp_path): - p1 = tmp_path / "empty1.txt" - p2 = tmp_path / "empty2.txt" - p1.write_bytes(b"") - p2.write_bytes(b"") - h1 = handler.handle(File(p1), hasher) - h2 = handler.handle(File(p2), hasher) - assert h1 == h2 - - def test_hash_matches_direct_sha256(self, handler, hasher, tmp_path): - """FileHandler must produce the same digest as BasicFileHasher(sha256) directly.""" - content = b"migration compatibility check" - p = tmp_path / "compat.txt" - p.write_bytes(content) - f = File(p) - handler_result = handler.handle(f, hasher) - direct_result = BasicFileHasher(algorithm="sha256").hash_file(p) - assert handler_result == direct_result - - def test_rejects_non_file_object(self, handler, hasher): - from pathlib import Path - with pytest.raises(TypeError, match="FileHandler"): - handler.handle(Path("/tmp"), hasher) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_file_handler.py -v -``` - -Expected: `ImportError` — `FileHandler` not defined yet. - -- [ ] **Step 3: Add `FileHandler` to `builtin_handlers.py`** - -In `src/orcapod/hashing/semantic_hashing/builtin_handlers.py`, add after the existing handler classes (before `register_builtin_python_type_handlers`): - -```python -class FileHandler: - """Hasher for ``orcapod.File`` objects — hashes file *content*. - - By the time ``handle`` is called, ``File``'s constructor has already validated - that the path exists and is a non-directory file (and is not a symlink when - ``follow_symlinks=False``). The hash is produced by reading file bytes through - the wrapped ``UPath``, which follows symlinks by default. - - Args: - file_hasher: Any object with a ``hash_file(path) -> ContentHash`` method. - """ - - def __init__(self, file_hasher: "FileContentHasherProtocol") -> None: - self.file_hasher = file_hasher - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: - # Deferred import breaks the circular dependency between this module and - # file_type.py — the same pattern used by ArrowTableHandler. - from orcapod.extension_types.file_type import File - if not isinstance(obj, File): - raise TypeError( - f"FileHandler: expected an orcapod.File, got {type(obj)!r}" - ) - logger.debug("FileHandler: hashing file content at %s", obj.__wrapped__) - return self.file_hasher.hash_file(obj.__wrapped__) -``` - -- [ ] **Step 4: Run the tests** - -```bash -uv run pytest tests/test_hashing/test_file_handler.py -v -``` - -Expected: All 7 tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py tests/test_hashing/test_file_handler.py -git commit -m "feat(hashing): add FileHandler for orcapod.File content hashing (ITL-450)" -``` - ---- - -## Task 5: Remove `PathHandler` and `UPathHandler` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Modify: `tests/test_hashing/test_extension_type_hashing.py` -- Modify: `tests/test_hashing/test_semantic_hasher.py` - -- [ ] **Step 1: Add the no-read test for `Path` columns** - -Open `tests/test_hashing/test_extension_type_hashing.py`. Append a new test class at the end of the file: - -```python -class TestPathColumnNoContentRead: - """After handler removal, hashing a Path column must not read file content.""" - - def test_path_column_hashing_does_not_read_file(self, ctx, tmp_path): - """Hashing a Path pointing at a nonexistent location raises no error. - - This verifies that PathHandler has been removed and Path columns hash - from their Arrow string content (the path string) without any file I/O. - """ - from pathlib import Path - from orcapod.hashing.visitors import SemanticHashingVisitor - - # Point at a path that definitely does not exist - nonexistent = tmp_path / "will_never_exist_abc123.txt" - assert not nonexistent.exists() - - arrow_type = ctx.type_converter.register_python_class(Path) - storage_val = ctx.type_converter.python_to_storage(nonexistent, Path) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - # Must not raise — no file read should occur - new_type, new_data = visitor.visit(arrow_type, storage_val) - - # Result is the raw Arrow storage (path string) — no content hash substitution - assert new_type == arrow_type - assert new_data == storage_val -``` - -- [ ] **Step 2: Run the new test to confirm it currently fails** - -```bash -uv run pytest tests/test_hashing/test_extension_type_hashing.py::TestPathColumnNoContentRead -v -``` - -Expected: `FAIL` — currently `PathHandler` reads the file, so hashing a nonexistent path raises `FileNotFoundError`. - -- [ ] **Step 3: Remove `PathHandler` and `UPathHandler` from `builtin_handlers.py`** - -In `src/orcapod/hashing/semantic_hashing/builtin_handlers.py`: - -1. Delete the `PathHandler` class entirely (lines defining `class PathHandler:` through its closing). -2. Delete the `UPathHandler` class entirely. -3. Remove these two lines from `register_builtin_python_type_handlers`: - ```python - registry.register(Path, PathHandler(file_hasher)) - registry.register(UPath, UPathHandler(file_hasher)) - ``` -4. Add the `File` registration in their place (after the `bytes`/`bytearray` registrations): - ```python - from orcapod.extension_types.file_type import File - registry.register(File, FileHandler(file_hasher)) - ``` -5. Remove the unused imports at the top of the file: - - `from pathlib import Path` ← remove (no longer used in this module) - - `from upath import UPath` ← remove (no longer used in this module) - -The module-level imports section should now look like: -```python -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any -from uuid import UUID - -from orcapod.types import ContentHash, PathLike, Schema - -if TYPE_CHECKING: - from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - FileContentHasherProtocol, - HandlerRegistryProtocol, - SemanticHasherProtocol, - ) -``` - -- [ ] **Step 4: Remove `PathHandler`-dependent tests from `test_semantic_hasher.py`** - -Search `tests/test_hashing/test_semantic_hasher.py` for any test that imports or directly uses `PathHandler` or `UPathHandler`. Remove those tests (or the `PathHandler`/`UPathHandler`-specific portions). Tests that use `Path` only for incidental file creation (not for testing content hashing via `PathHandler`) can stay. - -Run to find what needs removing: - -```bash -grep -n "PathHandler\|UPathHandler\|hash_object.*Path\|Path.*hash_object" tests/test_hashing/test_semantic_hasher.py -``` - -Delete or rewrite any test that asserts `Path` objects are content-hashed. Leave tests that use `tmp_path` only for creating real files passed to `File` or other handlers. - -- [ ] **Step 5: Port existing Path-column content-hash tests to use `File`** - -In `tests/test_hashing/test_extension_type_hashing.py`, find every test in `TestSemanticHashingVisitorExtension` and `TestCrossPathConsistency` that uses `Path` as the type for content-hashing assertions. Replace `Path` with `File`: - -```python -# Before: -arrow_type = ctx.type_converter.register_python_class(Path) -storage_val = ctx.type_converter.python_to_storage(Path(file), Path) - -# After: -from orcapod.extension_types.file_type import File -arrow_type = ctx.type_converter.register_python_class(File) -storage_val = ctx.type_converter.python_to_storage(File(file), File) -``` - -Also update the `test_binary_encoding_format` assertion — the type prefix will now be `b"orcapod:file"` instead of `b"orcapod:path"`: - -```python -# Before: -assert type_prefix == b"orcapod:path" - -# After: -assert type_prefix == b"orcapod:file" -``` - -- [ ] **Step 6: Run the full test suite** - -```bash -uv run pytest tests/test_hashing/ tests/test_extension_types/ -v -``` - -Expected: All tests pass. The `TestPathColumnNoContentRead` test that was failing in Step 2 now passes. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py \ - tests/test_hashing/test_extension_type_hashing.py \ - tests/test_hashing/test_semantic_hasher.py -git commit -m "refactor(hashing): remove PathHandler/UPathHandler; Path and UPath now string-hash (ITL-450)" -``` - ---- - -## Task 6: Update `v0.1.json` configuration - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Edit `v0.1.json`** - -Open `src/orcapod/contexts/data/v0.1.json` and make three changes: - -**Change 1:** In the `"logical_types"` array, add `LogicalFile` after `LogicalUUID`: - -```json -{ - "_class": "orcapod.extension_types.file_type.LogicalFile", - "_config": {} -} -``` - -**Change 2:** In the `"handlers"` array, remove these two entries: - -```json -[{"_type": "pathlib.Path"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PathHandler", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], -[{"_type": "upath.core.UPath"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.UPathHandler", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], -``` - -**Change 3:** In the `"handlers"` array, add the `File` handler entry (after the `bytearray` entry): - -```json -[{"_type": "orcapod.extension_types.file_type.File"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.FileHandler", "_config": {"file_hasher": {"_ref": "file_hasher"}}}], -``` - -- [ ] **Step 2: Run the integration tests that exercise the full context** - -```bash -uv run pytest tests/ -v -k "context or default_context or data_context or v0.1" -``` - -Expected: All pass. The context loads correctly with the new config. - -- [ ] **Step 3: Run the full test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: All tests pass. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json -git commit -m "feat(config): register LogicalFile and FileHandler; remove Path/UPath handlers in v0.1.json (ITL-450)" -``` - ---- - -## Task 7: Update exports and public API - -**Files:** -- Modify: `src/orcapod/__init__.py` -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `src/orcapod/hashing/semantic_hashing/__init__.py` - -- [ ] **Step 1: Add `File` to `src/orcapod/__init__.py`** - -Open `src/orcapod/__init__.py`. Add the import alongside the existing stable type aliases: - -```python -# Before the existing aliases: -from pathlib import Path -from upath import UPath -from uuid import UUID - -# Add: -from orcapod.extension_types.file_type import File -``` - -Add `"File"` to `__all__`: - -```python -__all__ = [ - # ... existing entries ... - # Stable type aliases - "File", # ← add this - "Path", - "UPath", - "UUID", -] -``` - -- [ ] **Step 2: Add `LogicalFile` to `src/orcapod/extension_types/__init__.py`** - -In `src/orcapod/extension_types/__init__.py`, add the import and `__all__` entry. Add after the existing `DataclassLogicalTypeFactory` / `PydanticLogicalTypeFactory` imports: - -```python -from .file_type import File, LogicalFile -``` - -Add to `__all__`: - -```python -# add after the PYDANTIC_CATEGORY block: -"File", -"LogicalFile", -``` - -- [ ] **Step 3: Update `src/orcapod/hashing/semantic_hashing/__init__.py`** - -In the imports block, replace: - -```python -from orcapod.hashing.semantic_hashing.builtin_handlers import ( - BytesHandler, - FunctionHandler, - PathHandler, - TypeObjectHandler, - UUIDHandler, - register_builtin_python_type_handlers, -) -``` - -with: - -```python -from orcapod.hashing.semantic_hashing.builtin_handlers import ( - BytesHandler, - FileHandler, - FunctionHandler, - TypeObjectHandler, - UUIDHandler, - register_builtin_python_type_handlers, -) -``` - -Update `__all__` — replace `"PathHandler"` with `"FileHandler"` and remove `"UPathHandler"` (it was never explicitly listed but verify): - -```python -__all__ = [ - "SemanticAwarePythonHasher", - "PythonTypeHandlerRegistry", - "BuiltinPythonTypeHandlerRegistry", - "ContentIdentifiableMixin", - "FileHandler", # ← replaces PathHandler - "UUIDHandler", - "BytesHandler", - "FunctionHandler", - "TypeObjectHandler", - "register_builtin_python_type_handlers", - "FunctionNameExtractor", - "FunctionSignatureExtractor", - "FunctionInfoExtractorFactory", -] -``` - -- [ ] **Step 4: Verify the public API smoke test** - -```bash -uv run python -c " -import orcapod -print('File:', orcapod.File) -print('Path:', orcapod.Path) -print('UPath:', orcapod.UPath) -print('UUID:', orcapod.UUID) -from orcapod.extension_types import LogicalFile -print('LogicalFile:', LogicalFile) -from orcapod.hashing.semantic_hashing import FileHandler -print('FileHandler:', FileHandler) -" -``` - -Expected output (no errors): -``` -File: -Path: -UPath: -UUID: -LogicalFile: -FileHandler: -``` - -- [ ] **Step 5: Run the full test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: All tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/__init__.py \ - src/orcapod/extension_types/__init__.py \ - src/orcapod/hashing/semantic_hashing/__init__.py -git commit -m "feat(api): export orcapod.File, LogicalFile, FileHandler; remove PathHandler/UPathHandler exports (ITL-450)" -``` - ---- - -## Task 8: CHANGELOG entry - -**Files:** -- Modify: `CHANGELOG.md` - -- [ ] **Step 1: Add the entry to `CHANGELOG.md`** - -Open `CHANGELOG.md` and add a new section at the top (or under the current `Unreleased` section if one exists): - -```markdown -## Unreleased - -### Added - -- `orcapod.File` — a content-identified, existence-validated file path type backed by - `upath.extensions.ProxyUPath`. Works with local, S3, GCS, and any other fsspec backend. - Constructed with `File(path)` (or `File(path, follow_symlinks=False)` to reject symlinks). - Hashes from file content (SHA-256); zero-byte files are valid. - -### Changed - -- **BREAKING:** `pathlib.Path` and `upath.UPath` columns no longer content-hash. They - now hash from the Arrow string content (the path string itself), with no file I/O. - This is identical to how `uuid.UUID` columns are hashed. Existing pipelines that - relied on `pathlib.Path` or `upath.UPath` for content-identified file columns **must - migrate to `orcapod.File`** — pipeline hashes will change otherwise. -``` - -- [ ] **Step 2: Commit** - -```bash -git add CHANGELOG.md -git commit -m "docs(changelog): add orcapod.File and breaking Path/UPath hash change (ITL-450)" -``` - ---- - -## Task 9: Final verification and push - -- [ ] **Step 1: Run the complete test suite one final time** - -```bash -uv run pytest tests/ -v --tb=short -``` - -Expected: All tests pass, no warnings about removed symbols. - -- [ ] **Step 2: Verify no stale references to `PathHandler` or `UPathHandler`** - -```bash -grep -rn "PathHandler\|UPathHandler" src/ tests/ -``` - -Expected: No matches (except possibly in comments or the CHANGELOG if mentioned by name). - -- [ ] **Step 3: Push the branch** - -```bash -git push -u origin eywalker/itl-450-add-opfile-type-and-refactor-pathlibpath-to-be-a-pure-path -``` - -- [ ] **Step 4: Create the PR** - -```bash -gh pr create \ - --base main \ - --title "feat(types): add op.File and make pathlib.Path/UPath pure-path types (ITL-450)" \ - --body "$(cat <<'EOF' -## Summary - -- Introduces `orcapod.File` as a content-identified, existence-validated file type backed by `upath.extensions.ProxyUPath` -- Adds `LogicalFile` Arrow extension type (`"orcapod.file"`, `large_string()` storage) -- Adds `FileHandler` for content hashing; removes `PathHandler` and `UPathHandler` -- `pathlib.Path` and `upath.UPath` now hash from path string (no file I/O) — **breaking change** - -Closes ITL-450 - -## Test plan - -- [ ] `tests/test_extension_types/test_file_type.py` — File constructor, LogicalFile roundtrip -- [ ] `tests/test_hashing/test_file_handler.py` — FileHandler content hash correctness -- [ ] `tests/test_hashing/test_extension_type_hashing.py` — Path no-read verification, File column hashing -- [ ] Full suite: `uv run pytest tests/ -v` - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - ---- - -## Self-Review - -**Spec coverage check:** - -| Spec requirement | Task | -|---|---| -| `op.File` as `ProxyUPath` subclass with constructor validation | Task 2 | -| `follow_symlinks` parameter, default True, raises on False+symlink | Task 2 | -| `_from_upath` override skips validation | Task 2 | -| `LogicalFile` with `"orcapod.file"` extension, `large_string()` storage | Task 3 | -| `python_to_storage` → path string | Task 3 | -| `storage_to_python` → `File(...)` re-validates | Task 3 | -| `FileHandler` with deferred import | Task 4 | -| `FileHandler` hashes via `file_hasher.hash_file(obj.__wrapped__)` | Task 4 | -| Remove `PathHandler` and `UPathHandler` | Task 5 | -| `Path` and `UPath` fall through to Arrow-content hashing | Task 5 | -| Update `v0.1.json` | Task 6 | -| `orcapod.File` export | Task 7 | -| `LogicalFile`, `FileHandler` exports | Task 7 | -| Remove `PathHandler`/`UPathHandler` exports | Task 7 | -| Constructor rejects nonexistent / dir / symlink (tests) | Task 2 | -| Zero-byte file hash (tests) | Task 4 | -| Hash matches `BasicFileHasher(sha256)` directly (tests) | Task 4 | -| Path column no-read test | Task 5 | -| CHANGELOG breaking-change note | Task 8 | - -All spec requirements covered. ✓ diff --git a/superpowers/plans/2026-06-30-release-workflow.md b/superpowers/plans/2026-06-30-release-workflow.md deleted file mode 100644 index a19bb7d53..000000000 --- a/superpowers/plans/2026-06-30-release-workflow.md +++ /dev/null @@ -1,425 +0,0 @@ -# Release Workflow Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the tag-push-triggered `publish.yml` with a `workflow_dispatch`-triggered `release.yml` that accepts a version input, runs pre-flight checks, builds the package, creates and pushes the release tag, publishes to TestPyPI then PyPI, creates a GitHub Release, and closes out the Linear release. - -**Architecture:** A single `release.yml` workflow replaces `publish.yml`. Pre-flight jobs (`test` matrix + `license-check`) run in parallel; `build` waits for all of them to succeed before creating the tag; publish jobs run sequentially (TestPyPI → PyPI); Linear jobs run independently and do not block the publish chain. - -**Tech Stack:** GitHub Actions, `uv`, `hatch-vcs`, OIDC Trusted Publishing, `linear/linear-release-action` - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `.github/workflows/release.yml` | **Create** | New `workflow_dispatch`-triggered release workflow | -| `.github/workflows/publish.yml` | **Delete** | Replaced by `release.yml` | -| `RELEASING.md` | **Modify** | Document the new `workflow_dispatch` release process | - -`release-sync.yml` and `_license-check.yml` are **not touched**. - ---- - -## Task 1: Create `release.yml` - -**Files:** -- Create: `.github/workflows/release.yml` - -- [ ] **Step 1: Write `release.yml`** - -Create `.github/workflows/release.yml` with the following content: - -```yaml -name: Release - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g. 0.1.0 or v0.1.0 — leading v is stripped automatically)' - required: true - type: string - -jobs: - test: - name: Test (Python ${{ matrix.python-version }}) - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - python-version: ["3.11", "3.12"] - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 # required: hatch-vcs needs full tag history - - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y graphviz libgraphviz-dev - - - name: Install dependencies - run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }} - - - name: Run tests - run: uv run --python ${{ matrix.python-version }} pytest -m "not postgres" --tb=short -q - - license-check: - uses: ./.github/workflows/_license-check.yml - - build: - name: Build distribution - needs: [test, license-check] - runs-on: ubuntu-latest - permissions: - contents: write # required to push the release tag - outputs: - version: ${{ steps.normalize.outputs.version }} - steps: - - name: Normalize version - id: normalize - run: | - VERSION="${{ inputs.version }}" - VERSION="${VERSION#v}" - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 # required: hatch-vcs reads tag to set version - - - name: Configure git identity - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Create local release tag - run: git tag "v${{ steps.normalize.outputs.version }}" - - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Build wheel and sdist - run: uv build - - - name: Guard against duplicate tag - run: | - if git ls-remote --tags origin "refs/tags/v${{ steps.normalize.outputs.version }}" | grep -q .; then - echo "::error::Tag v${{ steps.normalize.outputs.version }} already exists on origin. Aborting." - exit 1 - fi - - - name: Push release tag - run: git push origin "v${{ steps.normalize.outputs.version }}" - - - name: Upload dist artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: dist - path: dist/ - if-no-files-found: error - - publish-testpypi: - name: Publish → TestPyPI - needs: build - runs-on: ubuntu-latest - environment: - name: testpypi - url: https://test.pypi.org/p/orcapod - permissions: - id-token: write # required for OIDC Trusted Publishing - steps: - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Download dist artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: dist - path: dist/ - - - name: Publish to TestPyPI - run: uv publish --publish-url https://test.pypi.org/legacy/ dist/* - - publish-pypi: - name: Publish → PyPI - needs: [build, publish-testpypi] - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/orcapod - permissions: - id-token: write # required for OIDC Trusted Publishing - contents: write # required for creating GitHub Release - steps: - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Download dist artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: dist - path: dist/ - - - name: Publish to PyPI - run: uv publish dist/* - - - name: Create GitHub Release - uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 - with: - tag_name: "v${{ needs.build.outputs.version }}" - generate_release_notes: true - files: dist/* - - linear-sync: - name: Sync Linear release - needs: build - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - name: Sync Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: sync - version: "v${{ needs.build.outputs.version }}" - - linear-complete: - name: Complete Linear release - needs: [publish-pypi, linear-sync] - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - name: Complete Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: complete - version: "v${{ needs.build.outputs.version }}" -``` - -- [ ] **Step 2: Validate YAML syntax** - -Run: -```bash -python3 -c "import yaml; yaml.safe_load(open('.github/workflows/release.yml')); print('YAML OK')" -``` - -Expected output: -``` -YAML OK -``` - -If this fails, the file has a YAML syntax error — fix before continuing. - -- [ ] **Step 3: Commit** - -```bash -git add .github/workflows/release.yml -git commit -m "ci: add workflow_dispatch release workflow (ITL-449)" -``` - ---- - -## Task 2: Delete `publish.yml` - -**Files:** -- Delete: `.github/workflows/publish.yml` - -- [ ] **Step 1: Delete the file** - -```bash -git rm .github/workflows/publish.yml -``` - -- [ ] **Step 2: Verify it is gone** - -```bash -ls .github/workflows/ -``` - -Expected: `_license-check.yml`, `release-sync.yml`, `release.yml`, `run-objective-tests.yml`, `run-postgres-tests.yml`, `run-tests.yml`, `tests.yml` — no `publish.yml`. - -- [ ] **Step 3: Commit** - -```bash -git commit -m "ci: remove tag-push publish.yml (replaced by release.yml)" -``` - ---- - -## Task 3: Update `RELEASING.md` - -**Files:** -- Modify: `RELEASING.md` - -- [ ] **Step 1: Replace `RELEASING.md` with the updated content** - -Replace the entire file with: - -```markdown -# Releasing Orcapod - -This document describes how to cut a release of `orcapod` to PyPI. - -## Branching Model - -- All development happens on feature branches off `main`. -- There are no long-lived branches (the historical `dev` branch has been retired). -- No back-merges are needed. - -## Cutting a Release - -Releases are triggered from the GitHub Actions UI — no manual `git tag` step required. - -1. **Merge your branch into `main`** — open a PR, get it reviewed, merge it. - -2. **Trigger the release workflow** — go to - **[Actions → Release → Run workflow](https://github.com/nauticalab/orcapod-python/actions/workflows/release.yml)** - in the GitHub UI, enter the version (e.g. `0.1.0`), and click **Run workflow**. - - A leading `v` is stripped automatically — `v0.1.0` and `0.1.0` both work. - -3. **CI takes over** — the workflow runs the following jobs automatically: - - ``` - test (3.11) ─┐ - test (3.12) ─┤─ build ─ publish-testpypi ─ publish-pypi ─ linear-complete - license-check┘ │ - linear-sync ────────────┘ - ``` - - - Pre-flight: tests on Python 3.11 and 3.12, plus license check (in parallel) - - Build: normalises version, creates local tag, builds wheel + sdist, pushes tag to origin - - TestPyPI: publishes to test.pypi.org first as a staging step - - PyPI: publishes to pypi.org and creates a GitHub Release with generated release notes - - Linear: syncs and completes the Linear release (Merged → Done) - -## Pre-releases - -Release candidates (e.g. `0.1.0rc1`) follow exactly the same path — enter the version in the -workflow input. PyPI handles the stable vs pre-release distinction natively: - -- `pip install orcapod` — installs the latest **stable** release only -- `pip install --pre orcapod` — installs the latest release including pre-releases - -## Tag Format - -| Release type | Tag format | Example | -|-------------|------------|---------| -| Stable | `vMAJOR.MINOR.PATCH` | `v0.1.0` | -| Release candidate | `vMAJOR.MINOR.PATCHrcN` | `v0.1.0rc1` | -| Alpha | `vMAJOR.MINOR.PATCHaN` | `v0.1.0a1` | -| Beta | `vMAJOR.MINOR.PATCHbN` | `v0.1.0b1` | - -The workflow input accepts any of the above (with or without a leading `v`). The version is -derived from the git tag by `hatch-vcs` (`dynamic = ["version"]` in `pyproject.toml`) — no -manual version bump is needed. - -## Linear Release System - -When a release is triggered, the CI automatically transitions all Linear issues in -**"Merged"** status (whose PRs were included in the release) to **"Done"**. This is -handled by two jobs in `release.yml` that call `linear/linear-release-action`, plus a -separate `release-sync.yml` that runs on every push to `main`. - -### How it works - -| Trigger | Workflow / Job | Action | Effect | -|---------|---------------|--------|--------| -| Push to `main` | `release-sync.yml / sync` | `sync` (no version) | Associates the merged PR with the open Linear release draft | -| Release workflow `build` succeeds | `release.yml / linear-sync` | `sync --version ` | Finalises the commit set for this release version in Linear | -| After `publish-pypi` succeeds | `release.yml / linear-complete` | `complete --version ` | Marks the release done in Linear; triggers Merged → Done | - -### Prerequisites - -Before cutting the first release with this system active, a workspace admin must: - -1. **Set the `LINEAR_ACCESS_KEY` repo secret** in `nauticalab/orcapod-python` GitHub - settings (`Settings → Secrets and variables → Actions → New repository secret`). - Use the same Linear API key as `metamorphic-brain/axon`. - -2. **Configure a Linear release** in the workspace for orcapod-python via the Linear UI: - set the release name, which statuses count as "included" (at minimum: Merged), and - the target state for the transition (Done). - -3. **Update PyPI/TestPyPI Trusted Publisher configs** — if the Trusted Publisher on - pypi.org or test.pypi.org still references `publish.yml` as the workflow file, update - it to `release.yml` before triggering the first release. - -If `LINEAR_ACCESS_KEY` is not set: the `release-sync` workflow runs but the Linear action -may fail; the `linear-sync` and `linear-complete` jobs in `release.yml` fail but do not -block the PyPI publish (they have no dependents in the publish chain). - -### Replicating for other orcapod repos - -1. Copy `.github/workflows/release-sync.yml` verbatim. -2. Add `linear-sync` and `linear-complete` jobs to the repo's release workflow, - adjusting `needs:` in `linear-complete` to point at whichever job creates the GitHub - Release. -3. Set the `LINEAR_ACCESS_KEY` repo secret. -4. Configure a Linear release in the workspace for the new repo. -``` - -- [ ] **Step 2: Verify the file looks correct** - -```bash -head -30 RELEASING.md -``` - -Expected: starts with `# Releasing Orcapod` and the new `workflow_dispatch`-based instructions. - -- [ ] **Step 3: Commit** - -```bash -git add RELEASING.md -git commit -m "docs: update RELEASING.md for workflow_dispatch release process" -``` - ---- - -## Self-Review - -### Spec coverage - -| Spec requirement | Task covering it | -|---|---| -| `release.yml` created, replaces `publish.yml` | Task 1, Task 2 | -| `workflow_dispatch` trigger with version input | Task 1, Step 1 | -| Pre-flight: test matrix (3.11, 3.12) + license-check in parallel | Task 1, Step 1 (`test` + `license-check` jobs, no `needs:` on either) | -| Build: normalize version, create local tag, build, push tag | Task 1, Step 1 (`build` job) | -| Duplicate tag guard before pushing | Task 1, Step 1 (guard step in `build`) | -| Publish to TestPyPI then PyPI | Task 1, Step 1 (`publish-testpypi`, `publish-pypi` jobs) | -| GitHub Release with generated notes | Task 1, Step 1 (`softprops/action-gh-release` in `publish-pypi`) | -| linear-sync after build (parallel to publish chain) | Task 1, Step 1 (`linear-sync` needs `build`) | -| linear-complete after publish-pypi and linear-sync | Task 1, Step 1 (`linear-complete` needs `[publish-pypi, linear-sync]`) | -| All actions pinned to SHA digests | Task 1, Step 1 (all `uses:` lines) | -| `publish.yml` deleted | Task 2 | -| `RELEASING.md` updated | Task 3 | -| PyPI Trusted Publisher note (rename publish.yml → release.yml) | Task 3, Step 1 | - -All spec requirements covered. ✓ - -### Placeholder scan - -No TBDs, TODOs, or incomplete sections. ✓ - -### Type/name consistency - -- `steps.normalize.outputs.version` referenced in `build` job steps — defined by the `normalize` step with `id: normalize`. ✓ -- `needs.build.outputs.version` used in `publish-pypi`, `linear-sync`, `linear-complete` — all have `needs: build` or transitively depend on it, and the `build` job declares `outputs: version`. ✓ -- Action digest hashes are consistent across the plan. ✓ diff --git a/superpowers/plans/2026-06-30-union-typed-function-inputs.md b/superpowers/plans/2026-06-30-union-typed-function-inputs.md deleted file mode 100644 index 2a5a7088c..000000000 --- a/superpowers/plans/2026-06-30-union-typed-function-inputs.md +++ /dev/null @@ -1,448 +0,0 @@ -# Union-Typed Function Inputs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix `FunctionPod` construction to accept union-typed input arguments (e.g. `x: str | Path`) and add tests confirming construction, correct stream binding, and mismatch rejection. - -**Architecture:** `_FunctionPodBase.__init__` calls `ensure_types_registered_for_schemas` on both input and output schemas. The fix modifies that method to walk union branches and register each non-`None` branch individually rather than forwarding the whole union type to `register_python_class` (which rejects it). Stream-binding validation via `check_schema_compatibility` / `beartype.door.is_subhint` already works correctly and requires no changes. - -**Tech Stack:** Python 3.12, PyArrow, beartype, uv (test runner: `uv run pytest`) - ---- - -## File Map - -| File | Action | Purpose | -|---|---|---| -| `src/orcapod/semantic_types/universal_converter.py` | Modify | Walk union branches in `ensure_types_registered_for_schemas`; add clarifying comment in `_register_python_class_impl` | -| `src/orcapod/core/function_pod.py` | Modify | Add inline comments explaining union-input acceptance contract | -| `tests/test_core/function_pod/test_union_typed_inputs.py` | Create | All tests for this feature | -| `DESIGN_ISSUES.md` | Modify | Add resolved bug entry | - ---- - -### Task 1: Check out the feature branch - -**Files:** none (git only) - -- [ ] **Step 1: Create and check out the branch** - -```bash -git checkout -b eywalker/itl-452-verify-union-typed-function-inputs-are-accepted-concrete-pod -``` - -- [ ] **Step 2: Verify you are on the correct branch** - -```bash -git branch --show-current -``` - -Expected output: -``` -eywalker/itl-452-verify-union-typed-function-inputs-are-accepted-concrete-pod -``` - ---- - -### Task 2: Write failing tests - -**Files:** -- Create: `tests/test_core/function_pod/test_union_typed_inputs.py` - -- [ ] **Step 1: Create the test file** - -`tests/test_core/function_pod/test_union_typed_inputs.py`: - -```python -""" -Tests for union-typed function inputs (ITL-452). - -Verifies: -- FunctionPod construction succeeds when an input arg is declared with a union type. -- Binding a concrete stream of a matching type succeeds and the input schema - reflects the concrete branch (accessible via pod_stream.upstreams[0].output_schema()). -- Binding a stream whose type is not a member of the union raises ValueError. -- Data processing works correctly for each concrete branch. -""" - -from __future__ import annotations - -from pathlib import Path - -import pyarrow as pa -import pytest - -from orcapod.contexts import get_default_context -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.streams import ArrowTableStream - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def make_str_stream(n: int = 2) -> ArrowTableStream: - """Stream with tag=id (int64), data=x (large_string / str).""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("x", pa.large_string(), nullable=False), - ]) - table = pa.table( - { - "id": pa.array(list(range(n)), type=pa.int64()), - "x": pa.array([f"value_{i}" for i in range(n)], type=pa.large_string()), - }, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -def make_path_stream(n: int = 2) -> ArrowTableStream: - """Stream with tag=id (int64), data=x (orcapod.path / Path).""" - ctx = get_default_context() - path_arrow_type = ctx.type_converter.python_type_to_arrow_type(Path) - storage = pa.array([f"/tmp/test_{i}" for i in range(n)], type=pa.large_string()) - path_array = pa.ExtensionArray.from_storage(path_arrow_type, storage) - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("x", path_arrow_type, nullable=False), - ]) - table = pa.table( - { - "id": pa.array(list(range(n)), type=pa.int64()), - "x": path_array, - }, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -def make_int_stream(n: int = 2) -> ArrowTableStream: - """Stream with tag=id (int64), data=x (int64) — incompatible with str | Path.""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("x", pa.int64(), nullable=False), - ]) - table = pa.table( - { - "id": pa.array(list(range(n)), type=pa.int64()), - "x": pa.array(list(range(n)), type=pa.int64()), - }, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -def make_union_pod() -> FunctionPod: - """FunctionPod whose input x accepts str | Path.""" - def union_fn(x: str | Path) -> str: - return str(x) - - return FunctionPod(PythonDataFunction(union_fn, output_keys="result")) - - -# --------------------------------------------------------------------------- -# Construction tests -# --------------------------------------------------------------------------- - - -class TestUnionInputConstruction: - def test_pod_construction_succeeds(self): - """Creating a FunctionPod with a union-typed input must not raise.""" - def union_fn(x: str | Path) -> str: - return str(x) - - # Must not raise ValueError about complex unions - pod = FunctionPod(PythonDataFunction(union_fn, output_keys="result")) - assert pod is not None - - def test_input_schema_preserves_union_type(self): - """The data function's input_data_schema must record the full union type.""" - def union_fn(x: str | Path) -> str: - return str(x) - - pod = FunctionPod(PythonDataFunction(union_fn, output_keys="result")) - assert pod.data_function.input_data_schema["x"] == str | Path - - -# --------------------------------------------------------------------------- -# Stream binding tests -# --------------------------------------------------------------------------- - - -class TestUnionInputStreamBinding: - def test_bind_str_stream_succeeds(self): - """Binding a str-typed stream to a str | Path pod must succeed.""" - pod = make_union_pod() - pod_stream = pod.process(make_str_stream()) - # The upstream stream's data schema must report the concrete str type - _, input_data_schema = pod_stream.upstreams[0].output_schema() - assert input_data_schema["x"] == str - - def test_bind_path_stream_succeeds(self): - """Binding a Path-typed stream to a str | Path pod must succeed.""" - pod = make_union_pod() - pod_stream = pod.process(make_path_stream()) - # The upstream stream's data schema must report the concrete Path type - _, input_data_schema = pod_stream.upstreams[0].output_schema() - assert input_data_schema["x"] == Path - - def test_bind_incompatible_type_raises(self): - """Binding a stream whose type is not in the union must raise ValueError.""" - pod = make_union_pod() - with pytest.raises(ValueError, match="not compatible"): - pod.process(make_int_stream()) - - -# --------------------------------------------------------------------------- -# Data processing tests -# --------------------------------------------------------------------------- - - -class TestUnionInputDataProcessing: - def test_process_str_input_yields_correct_output(self): - """Processing a str-typed stream through a str | Path pod gives correct results.""" - def union_fn(x: str | Path) -> str: - return str(x).upper() - - pod = FunctionPod(PythonDataFunction(union_fn, output_keys="result")) - pod_stream = pod.process(make_str_stream(n=2)) - results = list(pod_stream.iter_data()) - assert len(results) == 2 - _, out_data = results[0] - assert out_data.as_dict()["result"] == "VALUE_0" - - def test_process_path_input_yields_correct_output(self): - """Processing a Path-typed stream through a str | Path pod gives correct results.""" - def union_fn(x: str | Path) -> str: - return str(x) - - pod = FunctionPod(PythonDataFunction(union_fn, output_keys="result")) - pod_stream = pod.process(make_path_stream(n=2)) - results = list(pod_stream.iter_data()) - assert len(results) == 2 - _, out_data = results[0] - assert out_data.as_dict()["result"] == "/tmp/test_0" -``` - -- [ ] **Step 2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_core/function_pod/test_union_typed_inputs.py -v -``` - -Expected: ALL tests FAIL — construction tests fail with: -``` -ValueError: Complex unions with multiple non-None types are not supported: str | pathlib.Path -``` - ---- - -### Task 3: Fix `ensure_types_registered_for_schemas` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` - -- [ ] **Step 1: Replace the body of `ensure_types_registered_for_schemas`** - -In `src/orcapod/semantic_types/universal_converter.py`, replace the existing method body (lines ~227–232): - -```python - def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: - """Ensure a LogicalType is registered for every annotation in schemas. - - Calls ``register_python_class`` for each annotation, which recursively - resolves nested types and synthesises via factory if needed. - When no ``LogicalTypeRegistry`` is configured, this is a no-op. - - Union-typed annotations (e.g. ``str | Path``) are handled by registering - each non-``None`` branch individually. Arrow has no union storage type, - so the union itself is never registered; instead each concrete branch's - ``LogicalType`` is made available so it is ready when a stream of that - type is bound. - - Args: - *schemas: One or more ``Schema`` mappings (column name → Python type). - - Raises: - TypeError: If a leaf class has no registered ``LogicalType`` and - no registered factory covers it. - """ - if self._logical_type_registry is None: - return - for schema in schemas: - for annotation in schema.values(): - origin = get_origin(annotation) - if origin is typing.Union or origin is types.UnionType: - # Union types (e.g. str | Path) are valid in function input - # schemas — they express that the pod accepts either concrete - # type. Register each non-None branch so its LogicalType is - # available when a stream is bound; the union itself has no - # Arrow representation. - for branch in get_args(annotation): - if branch is not type(None): - self.register_python_class(branch) - else: - self.register_python_class(annotation) -``` - -- [ ] **Step 2: Add a note at the complex-union raise in `_register_python_class_impl`** - -Locate the `raise ValueError` for complex unions in `_register_python_class_impl` (~line 295) and add a preceding comment: - -```python - # Optional[T] / T | None → strip None arm - if origin is typing.Union or origin is _types_mod.UnionType: - non_none = [a for a in args if a is not type(None)] - if len(non_none) == 1: - return self.register_python_class(non_none[0]) - # Direct callers must not pass complex unions here — there is no - # Arrow type for str | Path. For schema-level registration - # (where union-typed input args are valid), use - # ensure_types_registered_for_schemas(), which registers each - # non-None branch individually. - raise ValueError( - f"Complex unions with multiple non-None types are not supported: " - f"{annotation!r}. Only Optional[T] (T | None) is allowed." - ) -``` - -- [ ] **Step 3: Run the tests to confirm they now pass** - -```bash -uv run pytest tests/test_core/function_pod/test_union_typed_inputs.py -v -``` - -Expected: ALL 7 tests PASS. - -- [ ] **Step 4: Run the full test suite to check for regressions** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all existing tests continue to pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py \ - tests/test_core/function_pod/test_union_typed_inputs.py -git commit -m "fix(types): accept union-typed function inputs in ensure_types_registered_for_schemas (ITL-452)" -``` - ---- - -### Task 4: Add documentation comments to `function_pod.py` - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` - -- [ ] **Step 1: Add inline comment to `_FunctionPodBase.__init__`** - -Locate the `ensure_types_registered_for_schemas` call in `_FunctionPodBase.__init__` (~line 78) and add a comment immediately above it: - -```python - # Union-typed input args (e.g. x: str | Path) are deliberately accepted - # at construction time. ensure_types_registered_for_schemas registers - # each non-None branch individually; the union is only resolved to a - # concrete branch when a stream is bound — see _validate_input_schema. - self.data_context.type_converter.ensure_types_registered_for_schemas( - data_function.input_data_schema, - data_function.output_data_schema, - ) -``` - -- [ ] **Step 2: Add inline comment to `_validate_input_schema`** - -Locate `_validate_input_schema` (~line 129) and add a comment inside the method: - -```python - def _validate_input_schema(self, input_schema: Schema) -> None: - expected_data_schema = self.data_function.input_data_schema - # When expected_data_schema contains a union type (e.g. str | Path), - # check_schema_compatibility uses beartype.door.is_subhint which accepts - # any concrete branch: is_subhint(str, str | Path) → True, - # is_subhint(int, str | Path) → False. - if not schema_utils.check_schema_compatibility( - input_schema, expected_data_schema - ): - raise ValueError( - f"Incoming data data type {input_schema} is not compatible with expected input schema {expected_data_schema}" - ) -``` - -- [ ] **Step 3: Run tests to verify no regression from comment-only edits** - -```bash -uv run pytest tests/test_core/function_pod/ -q -``` - -Expected: all function_pod tests pass. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/core/function_pod.py -git commit -m "docs(function_pod): document union-typed input acceptance and bind-time resolution (ITL-452)" -``` - ---- - -### Task 5: Update DESIGN_ISSUES.md - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Add a resolved entry** - -Add the following new section to `DESIGN_ISSUES.md` (insert after the last resolved entry, before any open entries, or at the bottom of the file — follow the existing ordering convention): - -```markdown -## `src/orcapod/semantic_types/universal_converter.py` - -### UC1 — Premature rejection of union-typed function inputs -**Status:** resolved -**Severity:** high -**Issue:** ITL-452 - -`ensure_types_registered_for_schemas` forwarded each schema annotation directly -to `register_python_class`, which rejects complex union types (``str | Path``) -because Arrow has no native union storage type. This caused ``FunctionPod`` -construction to fail with a ``ValueError`` whenever a function declared a -union-typed input argument, even though union inputs are semantically valid -(they express that the pod accepts either concrete type, with the concrete -branch resolved at stream-binding time). - -**Fix:** Modified `ensure_types_registered_for_schemas` to detect union -annotations and register each non-``None`` branch individually, leaving -``register_python_class`` unchanged (it correctly rejects unions when invoked -directly for explicit type conversion). -``` - -- [ ] **Step 2: Commit** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs(design-issues): log UC1 union-typed input premature rejection as resolved (ITL-452)" -``` - ---- - -### Task 6: Final verification and push - -- [ ] **Step 1: Run the full test suite one final time** - -```bash -uv run pytest tests/ -q -``` - -Expected: all tests pass, no failures. - -- [ ] **Step 2: Push the branch** - -```bash -git push -u origin eywalker/itl-452-verify-union-typed-function-inputs-are-accepted-concrete-pod -``` diff --git a/superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md b/superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md deleted file mode 100644 index 8faa0f995..000000000 --- a/superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md +++ /dev/null @@ -1,764 +0,0 @@ -# SpikeInterface BaseRecording LogicalType — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `LogicalSIRecording` so that pods can accept and return SpikeInterface `BaseRecording` objects natively — stored as SI's JSON dump in Arrow, content-hashed by SHA-256 of that JSON. - -**Architecture:** `spikeinterface_types.py` imports `BaseRecording` at module level (raises `ImportError` if SI absent, caught gracefully by `extension_types/__init__.py`). `LogicalSIRecording` stores `to_dict(recursive=True)` as JSON `large_string`. `SIRecordingHandler` hashes those same JSON bytes via SHA-256. Registration into the default context is explicit via `register_spikeinterface_types()`. No changes to `v0.1.json` or `builtin_handlers.py`. - -**Tech Stack:** SpikeInterface ≥0.101, PyArrow, Polars, Python 3.12, pytest, uv - ---- - -## File Map - -| Action | Path | Responsibility | -|--------|------|----------------| -| Create | `src/orcapod/extension_types/spikeinterface_types.py` | `LogicalSIRecording`, `SIRecordingHandler`, `register_spikeinterface_types()` | -| Modify | `src/orcapod/extension_types/__init__.py` | Conditionally export `LogicalSIRecording`, `register_spikeinterface_types` | -| Modify | `pyproject.toml` | Add `spikeinterface` optional extras group | -| Create | `tests/test_extension_types/test_spikeinterface_types.py` | All SI type tests | - -`builtin_handlers.py` and `contexts/data/v0.1.json` are **not touched**. - ---- - -## Task 1: Add optional extras group to `pyproject.toml` - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add the extras group** - -In `pyproject.toml`, find the `[project.optional-dependencies]` section and make these two changes: - -```toml -spikeinterface = ["spikeinterface>=0.101"] -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]", "orcapod[spikeinterface]"] -``` - -The `all` line replaces the existing `all` line (add `"orcapod[spikeinterface]"` to the end of the existing list). - -- [ ] **Step 2: Verify the extras install correctly** - -```bash -uv run --with spikeinterface python -c "import spikeinterface; print(spikeinterface.__version__)" -``` - -Expected output: a version string like `0.104.8` (no errors). - -- [ ] **Step 3: Commit** - -```bash -git add pyproject.toml -git commit -m "chore(deps): add spikeinterface optional extras group (ITL-459)" -``` - ---- - -## Task 2: Create `spikeinterface_types.py` with `LogicalSIRecording` - -**Files:** -- Create: `src/orcapod/extension_types/spikeinterface_types.py` -- Create: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Write the failing import test** - -```python -# tests/test_extension_types/test_spikeinterface_types.py -"""Tests for LogicalSIRecording and SIRecordingHandler (ITL-459).""" - -from __future__ import annotations - -import json - -import numpy as np -import pytest - - -def test_spikeinterface_not_installed_raises_import_error(monkeypatch): - """Importing spikeinterface_types when SI is absent raises ImportError.""" - import sys - import importlib - - # Remove any cached spikeinterface modules - si_keys = [k for k in sys.modules if k == "spikeinterface" or k.startswith("spikeinterface.")] - saved = {k: sys.modules.pop(k) for k in si_keys} - # Block re-import - monkeypatch.setitem(sys.modules, "spikeinterface", None) - monkeypatch.setitem(sys.modules, "spikeinterface.core", None) - - try: - # Remove the cached spikeinterface_types module so it re-imports - si_types_key = "orcapod.extension_types.spikeinterface_types" - saved_si_types = sys.modules.pop(si_types_key, None) - with pytest.raises(ImportError, match="pip install orcapod\\[spikeinterface\\]"): - importlib.import_module(si_types_key) - finally: - # Restore everything - for k in list(sys.modules): - if k == "spikeinterface" or k.startswith("spikeinterface."): - del sys.modules[k] - sys.modules.update(saved) - if saved_si_types is not None: - sys.modules[si_types_key] = saved_si_types - else: - sys.modules.pop(si_types_key, None) - importlib.import_module(si_types_key) - - -# All tests below require spikeinterface — skip if not installed -si = pytest.importorskip("spikeinterface", reason="spikeinterface not installed") -import spikeinterface.core as si_core # noqa: E402 - - -def _make_numpy_recording() -> si_core.NumpyRecording: - """Create a small in-memory NumpyRecording for use as a test source.""" - rng = np.random.default_rng(42) - traces = rng.standard_normal((200, 4)).astype("float32") - return si_core.NumpyRecording([traces], sampling_frequency=30_000) - - -def test_logical_si_recording_importable(): - from orcapod.extension_types.spikeinterface_types import LogicalSIRecording - import pyarrow as pa - - lt = LogicalSIRecording() - assert lt.logical_type_name == "spikeinterface.recording" - assert lt.python_type is si_core.BaseRecording - assert lt.get_arrow_extension_type().storage_type == pa.large_string() -``` - -- [ ] **Step 2: Run to confirm failure** - -```bash -uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_recording_importable -v -``` - -Expected: `FAILED` — `ModuleNotFoundError: No module named 'orcapod.extension_types.spikeinterface_types'` - -- [ ] **Step 3: Create `spikeinterface_types.py`** - -```python -# src/orcapod/extension_types/spikeinterface_types.py -"""SpikeInterface LogicalType and handler for orcapod (ITL-459). - -``LogicalSIRecording`` maps ``spikeinterface.core.BaseRecording`` ↔ Arrow -``large_string`` using SpikeInterface's own ``to_dict(recursive=True)`` JSON -dump as the storage envelope. ``SIRecordingHandler`` hashes the same JSON -bytes via SHA-256 for content identity. - -This module requires the optional ``spikeinterface`` extras group:: - - pip install orcapod[spikeinterface] - -Register SI types into the default orcapod context before using them in -pods:: - - from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - register_spikeinterface_types() -""" - -from __future__ import annotations - -import hashlib -import json -import logging -from typing import TYPE_CHECKING, Any - -import polars as pl -import pyarrow as pa - -from orcapod.extension_types.base_logical_type import BaseLogicalType -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type -from orcapod.types import ContentHash - -if TYPE_CHECKING: - from orcapod.extension_types.protocols import TypeConverterProtocol - from orcapod.protocols.hashing_protocols import SemanticHasherProtocol - -try: - from spikeinterface.core import BaseRecording -except ImportError as _exc: - raise ImportError( - "spikeinterface is not installed. " - "Install it with: pip install orcapod[spikeinterface]" - ) from _exc - -logger = logging.getLogger(__name__) - - -class LogicalSIRecording(BaseLogicalType): - """Logical type for ``spikeinterface.core.BaseRecording``. - - Stores ``BaseRecording`` instances as Arrow ``large_string`` columns - tagged with extension name ``"spikeinterface.recording"``. The stored - value is SpikeInterface's own ``to_dict(recursive=True)`` output, - JSON-serialised. Loading reconstructs the recording via - ``spikeinterface.core.load_extractor(dict)``. - - Only recordings whose ``check_serializability("json")`` returns ``True`` - are accepted. Lazy recordings built on top of file-backed data (zarr, - binary folder, etc.) qualify. In-memory ``NumpyRecording`` objects do - not and raise ``ValueError`` with clear save instructions. - - Example: - >>> import tempfile, numpy as np - >>> import spikeinterface.core as si - >>> from orcapod.extension_types.spikeinterface_types import LogicalSIRecording - >>> lt = LogicalSIRecording() - >>> with tempfile.TemporaryDirectory() as tmp: - ... rec = si.NumpyRecording([np.zeros((100, 4), dtype="float32")], 30000) - ... saved = rec.save_to_folder(tmp + "/rec") - ... storage = lt.python_to_storage(saved) - ... recovered = lt.storage_to_python(storage) - ... saved.get_traces(segment_index=0).shape == recovered.get_traces(segment_index=0).shape - True - """ - - _arrow_ext_class = make_arrow_extension_type("spikeinterface.recording", pa.large_string()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("spikeinterface.recording", pa.large_string()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "spikeinterface.recording" - python_type: type = BaseRecording - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``BaseRecording``. - - Returns: - A ``pa.ExtensionType`` with extension name - ``"spikeinterface.recording"`` and storage type ``pa.large_string()``. - """ - if LogicalSIRecording._arrow_ext is None: - LogicalSIRecording._arrow_ext = LogicalSIRecording._arrow_ext_class() - return LogicalSIRecording._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``BaseRecording``. - - Returns: - A ``pl.BaseExtension`` registered under ``"spikeinterface.recording"``. - """ - if LogicalSIRecording._polars_ext is None: - LogicalSIRecording._polars_ext = LogicalSIRecording._polars_ext_class() - return LogicalSIRecording._polars_ext - - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None = None - ) -> str: - """Serialise a ``BaseRecording`` to its JSON storage representation. - - Args: - value: A ``BaseRecording`` instance whose - ``check_serializability("json")`` returns ``True``. - converter: Ignored. Present for protocol conformance. - - Returns: - A JSON string produced by ``recording.to_dict(recursive=True)``. - - Raises: - ValueError: If the recording is not JSON-serialisable (e.g. an - in-memory ``NumpyRecording``). - """ - if not value.check_serializability("json"): - raise ValueError( - "This BaseRecording is not JSON-serializable and cannot be stored " - "by orcapod. This typically means it holds data in memory (e.g. " - "NumpyRecording). Lazy recordings built on top of file-backed data " - "(zarr, binary folder, etc.) are fine and do not need to be " - "materialized first. If your recording is in-memory, call " - "recording.save_to_zarr(path) or recording.save_to_folder(path) " - "first, then pass the returned extractor to the pod." - ) - return json.dumps(value.to_dict(recursive=True)) - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None = None - ) -> BaseRecording: - """Reconstruct a ``BaseRecording`` from its JSON storage string. - - Args: - storage_value: A JSON string as stored in Arrow. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``BaseRecording`` instance reconstructed via - ``spikeinterface.core.load_extractor``. - - Raises: - ValueError: If ``storage_value`` is not valid JSON. - FileNotFoundError: If the backing zarr/folder no longer exists - (raised by SpikeInterface, propagated as-is). - """ - from spikeinterface.core import load_extractor - try: - si_dict = json.loads(storage_value) - except (json.JSONDecodeError, TypeError) as exc: - raise ValueError( - f"LogicalSIRecording: cannot deserialise storage value " - f"{storage_value!r}; expected a JSON string." - ) from exc - return load_extractor(si_dict) - - -class SIRecordingHandler: - """Semantic hash handler for ``spikeinterface.core.BaseRecording``. - - Computes a SHA-256 ``ContentHash`` of the JSON bytes produced by - ``recording.to_dict(recursive=True)``. This is identical to the bytes - that ``LogicalSIRecording`` stores in Arrow, so hash input and storage - representation are always consistent. - - The ``hasher`` argument is accepted for protocol conformance but not used — - hashing is done directly via ``hashlib.sha256`` to avoid overhead. - - Phase 2 (deferred — ITL-467): hash will additionally cover backing source - directory contents once efficient directory hashing infrastructure lands. - """ - - def handle(self, obj: Any, hasher: SemanticHasherProtocol | None) -> ContentHash: - """Return a SHA-256 ``ContentHash`` of the recording's JSON dump. - - Args: - obj: A ``BaseRecording`` instance. - hasher: Accepted for protocol conformance; not used. - - Returns: - A ``ContentHash`` with ``method="sha256"`` and digest equal to the - SHA-256 of ``json.dumps(recording.to_dict(recursive=True)).encode()``. - - Raises: - TypeError: If ``obj`` is not a ``BaseRecording``. - ValueError: If the recording is not JSON-serialisable (in-memory). - """ - if not isinstance(obj, BaseRecording): - raise TypeError( - f"SIRecordingHandler: expected BaseRecording, got {type(obj)!r}" - ) - if not obj.check_serializability("json"): - raise ValueError( - "Cannot hash an in-memory BaseRecording " - "(check_serializability('json') is False). " - "Save it to disk first with save_to_zarr() or save_to_folder()." - ) - json_bytes = json.dumps(obj.to_dict(recursive=True)).encode() - logger.debug("SIRecordingHandler: hashing %d JSON bytes", len(json_bytes)) - return ContentHash( - method="sha256", - digest=hashlib.sha256(json_bytes).digest(), - ) - - -def register_spikeinterface_types(context: Any = None) -> None: - """Register SpikeInterface LogicalTypes into an orcapod ``DataContext``. - - Call this once at startup, before any pods that use SpikeInterface types - are declared or executed. If ``context`` is ``None``, the default context - (from ``orcapod.contexts.get_default_context()``) is used. - - Args: - context: A ``DataContext`` instance, or ``None`` to use the default. - - Example: - >>> from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - >>> register_spikeinterface_types() # registers into the default context - """ - if context is None: - from orcapod.contexts import get_default_context - context = get_default_context() - - lt = LogicalSIRecording() - context.type_converter._logical_type_registry.register_logical_type(lt) - context.semantic_hasher.type_handler_registry.register(BaseRecording, SIRecordingHandler()) - logger.debug( - "register_spikeinterface_types: registered LogicalSIRecording and SIRecordingHandler" - ) -``` - -- [ ] **Step 4: Run both import tests** - -```bash -uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_recording_importable tests/test_extension_types/test_spikeinterface_types.py::test_spikeinterface_not_installed_raises_import_error -v -``` - -Expected: both `PASSED` - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add LogicalSIRecording, SIRecordingHandler, register_spikeinterface_types (ITL-459)" -``` - ---- - -## Task 3: `python_to_storage` — in-memory recording raises `ValueError` - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Add the test** - -Add to `tests/test_extension_types/test_spikeinterface_types.py` (after the `importorskip` line): - -```python -def test_in_memory_recording_raises(): - """NumpyRecording (json=False) raises ValueError with clear instructions.""" - from orcapod.extension_types.spikeinterface_types import LogicalSIRecording - - rec = _make_numpy_recording() - lt = LogicalSIRecording() - - with pytest.raises(ValueError, match="not JSON-serializable"): - lt.python_to_storage(rec) - - # The error message must distinguish in-memory from lazy file-backed recordings - with pytest.raises(ValueError, match="file-backed"): - lt.python_to_storage(rec) -``` - -- [ ] **Step 2: Run to confirm it passes (implementation already present in the skeleton)** - -```bash -uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_in_memory_recording_raises -v -``` - -Expected: `PASSED` - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_spikeinterface_types.py -git commit -m "test(spikeinterface_types): in-memory NumpyRecording raises ValueError (ITL-459)" -``` - ---- - -## Task 4: Round-trip tests — folder-backed, zarr-backed, ephemeral - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Add the three round-trip tests** - -Add to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_folder_recording_round_trip(tmp_path): - """Binary-folder-backed recording round-trips through python_to_storage / storage_to_python.""" - from orcapod.extension_types.spikeinterface_types import LogicalSIRecording - - saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) - lt = LogicalSIRecording() - - storage = lt.python_to_storage(saved) - assert isinstance(storage, str) - data = json.loads(storage) - assert "class" in data # SI dict always has a "class" key - - recovered = lt.storage_to_python(storage) - np.testing.assert_array_equal( - saved.get_traces(segment_index=0), - recovered.get_traces(segment_index=0), - ) - - -def test_zarr_recording_round_trip(tmp_path): - """Zarr-backed recording round-trips through python_to_storage / storage_to_python.""" - from orcapod.extension_types.spikeinterface_types import LogicalSIRecording - - saved = _make_numpy_recording().save_to_zarr(str(tmp_path / "rec.zarr")) - lt = LogicalSIRecording() - - storage = lt.python_to_storage(saved) - assert isinstance(storage, str) - - recovered = lt.storage_to_python(storage) - np.testing.assert_array_equal( - saved.get_traces(segment_index=0), - recovered.get_traces(segment_index=0), - ) - - -def test_ephemeral_recording_round_trip(tmp_path): - """A lazy preprocessing chain on a file-backed recording round-trips correctly. - - The preprocessed recording is NOT materialized to zarr/folder but IS - JSON-serializable because its root is file-backed. On reload, - SpikeInterface re-applies the preprocessing chain lazily. - """ - import spikeinterface.preprocessing as spre - from orcapod.extension_types.spikeinterface_types import LogicalSIRecording - - # Save the source to disk; apply a lazy preprocessing step on top - source = _make_numpy_recording().save_to_folder(str(tmp_path / "source")) - preprocessed = spre.bandpass_filter(source, freq_min=300.0, freq_max=6000.0) - - assert preprocessed.check_serializability("json"), ( - "Preprocessed recording on folder-backed source must be JSON-serializable" - ) - - lt = LogicalSIRecording() - storage = lt.python_to_storage(preprocessed) - recovered = lt.storage_to_python(storage) - - np.testing.assert_array_equal( - preprocessed.get_traces(segment_index=0), - recovered.get_traces(segment_index=0), - ) -``` - -- [ ] **Step 2: Run all three round-trip tests** - -```bash -uv run --with spikeinterface pytest \ - tests/test_extension_types/test_spikeinterface_types.py::test_folder_recording_round_trip \ - tests/test_extension_types/test_spikeinterface_types.py::test_zarr_recording_round_trip \ - tests/test_extension_types/test_spikeinterface_types.py::test_ephemeral_recording_round_trip \ - -v -``` - -Expected: all three `PASSED` - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_spikeinterface_types.py -git commit -m "test(spikeinterface_types): round-trip tests for folder, zarr, ephemeral recordings (ITL-459)" -``` - ---- - -## Task 5: Hash stability and content-change tests - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Add hash tests** - -Add to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_si_recording_handler_hash_stability(tmp_path): - """Same recording produces identical ContentHash across two calls.""" - from orcapod.extension_types.spikeinterface_types import SIRecordingHandler - from orcapod.types import ContentHash - - saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) - handler = SIRecordingHandler() - - h1 = handler.handle(saved, hasher=None) - h2 = handler.handle(saved, hasher=None) - - assert isinstance(h1, ContentHash) - assert h1 == h2 - - -def test_si_recording_handler_hash_changes_with_content(tmp_path): - """Different recordings produce different ContentHash values.""" - from orcapod.extension_types.spikeinterface_types import SIRecordingHandler - - rng = np.random.default_rng(0) - rec_a = si_core.NumpyRecording( - [rng.standard_normal((200, 4)).astype("float32")], sampling_frequency=30_000 - ).save_to_folder(str(tmp_path / "rec_a")) - rec_b = si_core.NumpyRecording( - [rng.standard_normal((200, 4)).astype("float32")], sampling_frequency=30_000 - ).save_to_folder(str(tmp_path / "rec_b")) - - handler = SIRecordingHandler() - assert handler.handle(rec_a, hasher=None) != handler.handle(rec_b, hasher=None) - - -def test_si_recording_handler_in_memory_raises(): - """SIRecordingHandler raises ValueError for in-memory recordings.""" - from orcapod.extension_types.spikeinterface_types import SIRecordingHandler - - rec = _make_numpy_recording() - handler = SIRecordingHandler() - with pytest.raises(ValueError, match="in-memory"): - handler.handle(rec, hasher=None) -``` - -- [ ] **Step 2: Run hash tests** - -```bash -uv run --with spikeinterface pytest \ - tests/test_extension_types/test_spikeinterface_types.py::test_si_recording_handler_hash_stability \ - tests/test_extension_types/test_spikeinterface_types.py::test_si_recording_handler_hash_changes_with_content \ - tests/test_extension_types/test_spikeinterface_types.py::test_si_recording_handler_in_memory_raises \ - -v -``` - -Expected: all three `PASSED` - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_spikeinterface_types.py -git commit -m "test(spikeinterface_types): hash stability and content-change tests (ITL-459)" -``` - ---- - -## Task 6: Integration test for `register_spikeinterface_types()` - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Add the integration test** - -Add to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_register_spikeinterface_types(tmp_path): - """register_spikeinterface_types() wires LogicalSIRecording and SIRecordingHandler - into the default context so they are found by type lookup.""" - from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - from orcapod.contexts import get_default_context - - register_spikeinterface_types() - ctx = get_default_context() - - saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) - - # LogicalType registered: type_converter can find it - arrow_type = ctx.type_converter.python_type_to_arrow_type(type(saved)) - assert arrow_type.extension_name == "spikeinterface.recording" - - # Handler registered: semantic_hasher can find it - from orcapod.types import ContentHash - handler = ctx.semantic_hasher.type_handler_registry.get_handler(saved) - assert handler is not None - result = handler.handle(saved, hasher=None) - assert isinstance(result, ContentHash) - - # Full round-trip through python_to_storage / storage_to_python - storage = ctx.type_converter.python_to_storage(saved, type(saved)) - recovered = ctx.type_converter.storage_to_python(storage, type(saved)) - np.testing.assert_array_equal( - saved.get_traces(segment_index=0), - recovered.get_traces(segment_index=0), - ) -``` - -- [ ] **Step 2: Run the integration test** - -```bash -uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_register_spikeinterface_types -v -``` - -Expected: `PASSED` - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_spikeinterface_types.py -git commit -m "test(spikeinterface_types): integration test for register_spikeinterface_types (ITL-459)" -``` - ---- - -## Task 7: Export from `extension_types/__init__.py` - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` - -- [ ] **Step 1: Add conditional import and export** - -In `src/orcapod/extension_types/__init__.py`, add after the `from .numpy_type import LogicalNumpyArray # ITL-460` line: - -```python -# ITL-459 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import LogicalSIRecording, register_spikeinterface_types - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -And at the end of `__all__`, add: - -```python - # ITL-459 (conditional — only present when spikeinterface is installed) - *( ["LogicalSIRecording", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), -``` - -- [ ] **Step 2: Verify import works with SI installed** - -```bash -uv run --with spikeinterface python -c " -from orcapod.extension_types import LogicalSIRecording, register_spikeinterface_types, __all__ -assert 'LogicalSIRecording' in __all__ -print('OK with SI:', LogicalSIRecording, register_spikeinterface_types) -" -``` - -Expected: prints the class and function without error. - -- [ ] **Step 3: Verify import works without SI installed** - -```bash -uv run python -c " -from orcapod.extension_types import __all__ -assert 'LogicalSIRecording' not in __all__, f'should not be exported, got: {__all__}' -print('OK without SI: LogicalSIRecording not in __all__') -" -``` - -Expected: prints the OK message without error. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "feat(extension_types): conditionally export LogicalSIRecording when SI available (ITL-459)" -``` - ---- - -## Task 8: Full suite check and changelog - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Run all spikeinterface tests** - -```bash -uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py -v -``` - -Expected: all 10 tests `PASSED` (1 not-installed test + 9 SI tests). - -- [ ] **Step 2: Run the full extension_types suite without SI to verify no regressions** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all pre-existing tests pass; SI tests that need SI are `SKIPPED` (not `FAILED`). The `test_spikeinterface_not_installed_raises_import_error` test runs and passes even without SI. - -- [ ] **Step 3: Run the full test suite** - -```bash -uv run pytest tests/ -v --timeout=60 -``` - -Expected: all pre-existing tests pass; new SI tests either `PASSED` (with SI) or `SKIPPED` (without SI). - -- [ ] **Step 4: Update `v0.1.json` changelog** - -In `src/orcapod/contexts/data/v0.1.json`, append to the `"changelog"` array: - -```json -"Added spikeinterface.BaseRecording as a native value type via LogicalSIRecording (large_string/JSON) and SIRecordingHandler (SHA-256 of JSON bytes); requires pip install orcapod[spikeinterface] and a register_spikeinterface_types() call (ITL-459)" -``` - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json -git commit -m "docs(context): record LogicalSIRecording in v0.1.json changelog (ITL-459)" -``` diff --git a/superpowers/plans/2026-07-01-itl-460-numpy-ndarray-native-value-type.md b/superpowers/plans/2026-07-01-itl-460-numpy-ndarray-native-value-type.md deleted file mode 100644 index 0ab2673ee..000000000 --- a/superpowers/plans/2026-07-01-itl-460-numpy-ndarray-native-value-type.md +++ /dev/null @@ -1,684 +0,0 @@ -# numpy.ndarray Native Value Type Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `numpy.ndarray` as a first-class orcapod value type so pods can return arrays directly without wrapping them in `op.File`. - -**Architecture:** Add `LogicalNumpyArray` (a new `BaseLogicalType` subclass) that maps `np.ndarray` ↔ Arrow `large_binary` using numpy's own `.npy` binary format, register it in the default context via `v0.1.json`, and add `NumpyArrayHandler` to the semantic hasher registry so arrays are content-hashed correctly. Object-dtype arrays are rejected eagerly at both write and hash time. - -**Tech Stack:** Python, PyArrow (`pa.large_binary`, extension types), Polars, numpy (`np.save`/`np.load`), pytest. - ---- - -## File Map - -| Action | Path | Responsibility | -|---|---|---| -| Create | `src/orcapod/extension_types/numpy_type.py` | `LogicalNumpyArray` class | -| Modify | `src/orcapod/extension_types/__init__.py` | Re-export `LogicalNumpyArray` | -| Modify | `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Add `NumpyArrayHandler` class | -| Modify | `src/orcapod/contexts/data/v0.1.json` | Register both in default context | -| Modify | `pyproject.toml` | Explicit numpy dependency | -| Create | `tests/test_extension_types/test_numpy_type.py` | Unit tests for `LogicalNumpyArray` | -| Create | `tests/test_hashing/test_numpy_handler.py` | Unit tests for `NumpyArrayHandler` | -| Modify | `tests/test_extension_types/test_roundtrips.py` | Integration round-trip test | - ---- - -## Task 1: Add `LogicalNumpyArray` with failing unit tests - -**Files:** -- Create: `tests/test_extension_types/test_numpy_type.py` -- Create: `src/orcapod/extension_types/numpy_type.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_extension_types/test_numpy_type.py`: - -```python -"""Tests for LogicalNumpyArray.""" -from __future__ import annotations - -import io - -import numpy as np -import pyarrow as pa -import pytest - -from orcapod.extension_types.protocols import LogicalTypeProtocol - - -class TestLogicalNumpyArrayProtocol: - def test_isinstance_logical_type(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - assert isinstance(LogicalNumpyArray(), LogicalTypeProtocol) - - def test_logical_type_name(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - assert LogicalNumpyArray().logical_type_name == "numpy.ndarray" - - def test_python_type(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - assert LogicalNumpyArray().python_type is np.ndarray - - def test_arrow_ext_name(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - assert LogicalNumpyArray().get_arrow_extension_type().extension_name == "numpy.ndarray" - - def test_arrow_ext_storage_type(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - assert LogicalNumpyArray().get_arrow_extension_type().storage_type == pa.large_binary() - - def test_arrow_ext_is_cached(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - lt = LogicalNumpyArray() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - def test_polars_ext_is_cached(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - lt = LogicalNumpyArray() - assert lt.get_polars_extension_type() is lt.get_polars_extension_type() - - -class TestLogicalNumpyArrayStorage: - def test_python_to_storage_returns_bytes(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - arr = np.array([1.0, 2.0, 3.0]) - result = LogicalNumpyArray().python_to_storage(arr) - assert isinstance(result, bytes) - - def test_object_dtype_raises_before_save(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - arr = np.array([1, "hello", None], dtype=object) - with pytest.raises(ValueError, match="object-dtype"): - LogicalNumpyArray().python_to_storage(arr) - - def test_structured_with_object_field_raises(self): - from orcapod.extension_types.numpy_type import LogicalNumpyArray - arr = np.zeros(3, dtype=[("label", object), ("value", np.int32)]) - with pytest.raises((ValueError, Exception)): - LogicalNumpyArray().python_to_storage(arr) - - -class TestLogicalNumpyArrayRoundTrip: - def _rt(self, arr: np.ndarray) -> np.ndarray: - from orcapod.extension_types.numpy_type import LogicalNumpyArray - lt = LogicalNumpyArray() - return lt.storage_to_python(lt.python_to_storage(arr)) - - def test_1d_float64(self): - arr = np.array([1.0, 2.0, 3.0], dtype=np.float64) - assert np.array_equal(self._rt(arr), arr) - assert self._rt(arr).dtype == arr.dtype - - def test_2d_int32(self): - arr = np.array([[1, 2], [3, 4]], dtype=np.int32) - result = self._rt(arr) - assert np.array_equal(result, arr) - assert result.shape == arr.shape - assert result.dtype == arr.dtype - - def test_3d_uint8(self): - arr = np.zeros((2, 3, 4), dtype=np.uint8) - result = self._rt(arr) - assert np.array_equal(result, arr) - assert result.shape == arr.shape - - def test_zero_size(self): - arr = np.array([], dtype=np.float32) - result = self._rt(arr) - assert np.array_equal(result, arr) - assert result.dtype == arr.dtype - - def test_single_element(self): - arr = np.array([42], dtype=np.int64) - result = self._rt(arr) - assert np.array_equal(result, arr) - - def test_structured_record_array(self): - dt = np.dtype([("x", np.float64), ("y", np.int32)]) - arr = np.array([(1.0, 2), (3.0, 4)], dtype=dt) - result = self._rt(arr) - assert result.dtype == arr.dtype - assert np.array_equal(result["x"], arr["x"]) - assert np.array_equal(result["y"], arr["y"]) - - def test_fortran_order(self): - arr = np.asfortranarray(np.array([[1, 2], [3, 4]], dtype=np.float64)) - result = self._rt(arr) - assert np.array_equal(result, arr) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_extension_types/test_numpy_type.py -v 2>&1 | head -30 -``` - -Expected: `ModuleNotFoundError` or `ImportError` — `numpy_type` does not exist yet. - -- [ ] **Step 3: Create `src/orcapod/extension_types/numpy_type.py`** - -```python -"""``numpy.ndarray`` logical type for orcapod. - -``LogicalNumpyArray`` maps ``numpy.ndarray`` ↔ Arrow ``large_binary`` using -numpy's own ``.npy`` binary format as the storage envelope. This preserves -dtype (including structured/record field names and types), shape, byte order, -and the raw data buffer for all array kinds: 1-D, 2-D, N-D, and structured. - -Object-dtype arrays (``dtype=object`` or structured dtypes containing object -fields) are explicitly rejected — they require pickling, which is disabled for -security. See the ``Limitations`` section in the ITL-460 design spec. -""" - -from __future__ import annotations - -import io -from typing import TYPE_CHECKING, Any - -import numpy as np -import polars as pl -import pyarrow as pa - -from orcapod.extension_types.base_logical_type import BaseLogicalType -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type - -if TYPE_CHECKING: - from orcapod.extension_types.protocols import TypeConverterProtocol - - -class LogicalNumpyArray(BaseLogicalType): - """Logical type for ``numpy.ndarray``. - - Stores arrays as Arrow ``large_binary`` using numpy's ``.npy`` on-disk - format as the storage envelope. The ``.npy`` format is self-describing: - it encodes dtype (including structured/record field names), shape, byte - order, and raw data — so no custom parsing is required and all array - kinds round-trip exactly. - - The extension name ``"numpy.ndarray"`` is chosen over the ``orcapod.*`` - convention because it is a highly stable, well-known name that makes - Arrow schemas immediately self-documenting and may enable interoperability - with other ecosystem tools that independently serialise numpy arrays. - - Object-dtype arrays are not supported. ``python_to_storage`` raises - ``ValueError`` immediately (before calling ``np.save``) when - ``array.dtype.kind == "O"``. ``np.save(..., allow_pickle=False)`` acts as - a secondary guard for structured dtypes that contain object-typed fields. - - Example: - >>> lt = LogicalNumpyArray() - >>> arr = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float64) - >>> recovered = lt.storage_to_python(lt.python_to_storage(arr)) - >>> np.array_equal(recovered, arr) - True - """ - - _arrow_ext_class = make_arrow_extension_type("numpy.ndarray", pa.large_binary()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("numpy.ndarray", pa.large_binary()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "numpy.ndarray" - python_type: type = np.ndarray - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``numpy.ndarray``. - - Returns: - A ``pa.ExtensionType`` instance with extension name - ``"numpy.ndarray"`` and storage type ``pa.large_binary()``. - """ - if LogicalNumpyArray._arrow_ext is None: - LogicalNumpyArray._arrow_ext = LogicalNumpyArray._arrow_ext_class() - return LogicalNumpyArray._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``numpy.ndarray``. - - Returns: - A ``pl.BaseExtension`` instance registered under - ``"numpy.ndarray"``. - """ - if LogicalNumpyArray._polars_ext is None: - LogicalNumpyArray._polars_ext = LogicalNumpyArray._polars_ext_class() - return LogicalNumpyArray._polars_ext - - def python_to_storage(self, value: Any, converter: TypeConverterProtocol | None = None) -> bytes: - """Serialise a ``numpy.ndarray`` to its ``.npy`` byte representation. - - Args: - value: A ``numpy.ndarray`` instance. Object-dtype arrays - (``dtype.kind == "O"``) are rejected immediately. - converter: Ignored. Present for protocol conformance. - - Returns: - Raw bytes in numpy's ``.npy`` format encoding dtype, shape, - byte order, and data. - - Raises: - ValueError: If ``value.dtype.kind == "O"`` (object-dtype array), - or if the array contains object-typed fields in a structured - dtype (caught by ``allow_pickle=False`` in ``np.save``). - """ - if value.dtype.kind == "O": - raise ValueError( - f"LogicalNumpyArray does not support object-dtype arrays " - f"(dtype={value.dtype!r}). Object arrays require pickling, " - "which is disabled for security. Serialise the array contents " - "manually (e.g. to bytes or a File) before passing to orcapod." - ) - buf = io.BytesIO() - np.save(buf, value, allow_pickle=False) - return buf.getvalue() - - def storage_to_python(self, storage_value: Any, converter: TypeConverterProtocol | None = None) -> np.ndarray: - """Reconstruct a ``numpy.ndarray`` from its ``.npy`` byte representation. - - Args: - storage_value: Raw bytes as stored in Arrow ``large_binary``. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``numpy.ndarray`` with dtype and shape as originally stored. - - Raises: - ValueError: If ``storage_value`` is not valid ``.npy`` bytes. - """ - buf = io.BytesIO(bytes(storage_value)) - return np.load(buf, allow_pickle=False) -``` - -- [ ] **Step 4: Run tests to confirm they pass** - -```bash -uv run pytest tests/test_extension_types/test_numpy_type.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/numpy_type.py tests/test_extension_types/test_numpy_type.py -git commit -m "feat(extension_types): add LogicalNumpyArray for numpy.ndarray (ITL-460)" -``` - ---- - -## Task 2: Add `NumpyArrayHandler` to semantic hasher - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Create: `tests/test_hashing/test_numpy_handler.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_hashing/test_numpy_handler.py`: - -```python -"""Tests for NumpyArrayHandler.""" -from __future__ import annotations - -import numpy as np -import pytest - -from orcapod.hashing.semantic_hashing.builtin_handlers import NumpyArrayHandler - - -class TestNumpyArrayHandler: - def test_returns_bytes(self): - handler = NumpyArrayHandler() - arr = np.array([1.0, 2.0, 3.0]) - result = handler.handle(arr, hasher=None) - assert isinstance(result, bytes) - - def test_same_content_same_bytes(self): - handler = NumpyArrayHandler() - arr = np.array([1, 2, 3], dtype=np.int32) - assert handler.handle(arr, None) == handler.handle(arr.copy(), None) - - def test_different_dtype_different_bytes(self): - handler = NumpyArrayHandler() - a = np.array([1, 2, 3], dtype=np.int32) - b = np.array([1, 2, 3], dtype=np.float64) - assert handler.handle(a, None) != handler.handle(b, None) - - def test_same_values_different_shape_different_bytes(self): - handler = NumpyArrayHandler() - a = np.array([1, 2, 3, 4], dtype=np.int32) - b = np.array([[1, 2], [3, 4]], dtype=np.int32) - assert handler.handle(a, None) != handler.handle(b, None) - - def test_structured_array_field_names_in_bytes(self): - handler = NumpyArrayHandler() - dt_xy = np.dtype([("x", np.float64), ("y", np.int32)]) - dt_ab = np.dtype([("a", np.float64), ("b", np.int32)]) - arr_xy = np.array([(1.0, 2)], dtype=dt_xy) - arr_ab = np.array([(1.0, 2)], dtype=dt_ab) - # Same values, same field types, different field names → different bytes - assert handler.handle(arr_xy, None) != handler.handle(arr_ab, None) - - def test_raises_type_error_for_non_ndarray(self): - handler = NumpyArrayHandler() - with pytest.raises(TypeError, match="numpy.ndarray"): - handler.handle([1, 2, 3], hasher=None) - - def test_raises_value_error_for_object_dtype(self): - handler = NumpyArrayHandler() - arr = np.array([1, "hello", None], dtype=object) - with pytest.raises(ValueError, match="object-dtype"): - handler.handle(arr, hasher=None) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_numpy_handler.py -v 2>&1 | head -20 -``` - -Expected: `ImportError` — `NumpyArrayHandler` does not exist yet. - -- [ ] **Step 3: Add `NumpyArrayHandler` to `builtin_handlers.py`** - -Add the following class at the end of the file, just before the `register_builtin_python_type_handlers` function (i.e. after `DirectoryHandler`): - -```python -class NumpyArrayHandler: - """Hasher for ``numpy.ndarray`` — content hash via numpy's ``.npy`` binary format. - - Returns the raw ``.npy`` bytes produced by ``np.save``, which encode dtype - (including structured/record field names), shape, byte order, and data in - numpy's stable on-disk format. This is identical to what ``LogicalNumpyArray`` - stores in Arrow, so the hash input and storage bytes are always consistent. - - Object-dtype arrays are rejected with ``ValueError`` immediately (before - ``np.save`` is called). Structured dtypes containing object-typed fields - are caught by ``allow_pickle=False`` in ``np.save``. - """ - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> bytes: - """Return the ``.npy`` bytes for ``obj``. - - Args: - obj: A ``numpy.ndarray`` instance. - hasher: Ignored. Present for protocol conformance. - - Returns: - Raw ``.npy`` bytes encoding dtype, shape, byte order, and data. - - Raises: - TypeError: If ``obj`` is not a ``numpy.ndarray``. - ValueError: If ``obj`` has ``dtype.kind == "O"`` (object dtype). - """ - import io - import numpy as np - if not isinstance(obj, np.ndarray): - raise TypeError( - f"NumpyArrayHandler: expected numpy.ndarray, got {type(obj)!r}" - ) - if obj.dtype.kind == "O": - raise ValueError( - f"NumpyArrayHandler does not support object-dtype arrays " - f"(dtype={obj.dtype!r}). Object arrays require pickling, " - "which is disabled for security." - ) - buf = io.BytesIO() - np.save(buf, obj, allow_pickle=False) - return buf.getvalue() -``` - -- [ ] **Step 4: Run tests to confirm they pass** - -```bash -uv run pytest tests/test_hashing/test_numpy_handler.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py tests/test_hashing/test_numpy_handler.py -git commit -m "feat(hashing): add NumpyArrayHandler for numpy.ndarray content hashing (ITL-460)" -``` - ---- - -## Task 3: Register in `v0.1.json` and export from `__init__.py` - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add `LogicalNumpyArray` to `v0.1.json` `logical_types` list** - -Open `src/orcapod/contexts/data/v0.1.json`. In the `logical_type_registry._config.logical_types` array, add the following entry after the `LogicalDirectory` entry: - -```json -{ - "_class": "orcapod.extension_types.numpy_type.LogicalNumpyArray", - "_config": {} -} -``` - -The full `logical_types` array should now end with: -```json -{ - "_class": "orcapod.extension_types.file_type.LogicalFile", - "_config": {} -}, -{ - "_class": "orcapod.extension_types.directory_type.LogicalDirectory", - "_config": {} -}, -{ - "_class": "orcapod.extension_types.numpy_type.LogicalNumpyArray", - "_config": {} -} -``` - -- [ ] **Step 2: Add `NumpyArrayHandler` to `v0.1.json` `handlers` list** - -In the same file, inside `python_type_handler_registry._config.handlers`, add the following entry (after the `pyarrow.RecordBatch` entry): - -```json -[{"_type": "numpy.ndarray"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.NumpyArrayHandler", "_config": {}}] -``` - -- [ ] **Step 3: Add `LogicalNumpyArray` to `extension_types/__init__.py`** - -Open `src/orcapod/extension_types/__init__.py`. Add the import after the `LogicalDirectory` import line: - -```python -from .numpy_type import LogicalNumpyArray # ITL-460 -``` - -Add `"LogicalNumpyArray"` to `__all__` after `"LogicalDirectory"`: - -```python - # ITL-451 - "LogicalDirectory", - # ITL-460 - "LogicalNumpyArray", -``` - -- [ ] **Step 4: Add numpy to `pyproject.toml` dependencies** - -Open `pyproject.toml`. In the `[project] dependencies` list, add: - -```toml -"numpy>=1.26.4", -``` - -Place it alongside the other scientific stack deps (near `pyarrow`, `polars`). - -- [ ] **Step 5: Verify the default context loads without error** - -```bash -uv run python -c " -from orcapod.contexts import get_default_context -ctx = get_default_context() -registry = ctx.type_converter._logical_type_registry -lt = registry.get_by_python_type(__import__('numpy').ndarray) -print('Registered logical type:', lt) -print('logical_type_name:', lt.logical_type_name) -" -``` - -Expected output: -``` -Registered logical type: -logical_type_name: numpy.ndarray -``` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json src/orcapod/extension_types/__init__.py pyproject.toml -git commit -m "feat(contexts): register LogicalNumpyArray and NumpyArrayHandler in default context (ITL-460)" -``` - ---- - -## Task 4: Integration round-trip test - -**Files:** -- Modify: `tests/test_extension_types/test_roundtrips.py` - -- [ ] **Step 1: Add the round-trip test** - -Open `tests/test_extension_types/test_roundtrips.py`. Add the following test function after `test_builtin_uuid_round_trip`: - -```python -def test_builtin_ndarray_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """numpy.ndarray round-trips through storage with extension name ``numpy.ndarray``. - - Tests both a simple 1-D float64 array and a structured (record) array with - named fields, since structured arrays are a primary motivation for this type. - The read-side converter already knows about ``numpy.ndarray`` because it is - registered in the default context (``v0.1.json``). - """ - import numpy as np - - arr_simple = np.array([1.0, 2.0, 3.0], dtype=np.float64) - arr_struct = np.array([(1.0, 10), (2.0, 20)], dtype=np.dtype([("x", np.float64), ("y", np.int32)])) - - # Simple 1-D float64 array - result, read_converter = _write_and_read( - {"col": np.ndarray}, - [{"col": arr_simple}], - storage_backend, - tmp_path / "simple", - ) - - field = result.schema.field("col") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'col', got plain type {field.type!r}" - ) - assert field.type.extension_name == "numpy.ndarray" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert isinstance(rows[0]["col"], np.ndarray) - assert np.array_equal(rows[0]["col"], arr_simple) - assert rows[0]["col"].dtype == arr_simple.dtype - - # Structured (record) array - result2, read_converter2 = _write_and_read( - {"col": np.ndarray}, - [{"col": arr_struct}], - storage_backend, - tmp_path / "struct", - ) - - rows2 = read_converter2.arrow_table_to_python_dicts(result2) - assert len(rows2) == 1 - recovered = rows2[0]["col"] - assert isinstance(recovered, np.ndarray) - assert recovered.dtype == arr_struct.dtype - assert np.array_equal(recovered["x"], arr_struct["x"]) - assert np.array_equal(recovered["y"], arr_struct["y"]) -``` - -Also add `import numpy as np` to the top-level imports section of the file if not already present (check first — the import is inside the test function above to be safe either way). - -- [ ] **Step 2: Run the round-trip test** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py::test_builtin_ndarray_round_trip -v -``` - -Expected: PASS for both `parquet` and `delta` backends. - -- [ ] **Step 3: Run the full test suite to check for regressions** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all existing tests continue to pass. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_extension_types/test_roundtrips.py -git commit -m "test(extension_types): add numpy.ndarray round-trip integration test (ITL-460)" -``` - ---- - -## Task 5: Final verification and push - -- [ ] **Step 1: Run the complete test suite one more time** - -```bash -uv run pytest tests/ -q -``` - -Expected: all tests pass, no failures. - -- [ ] **Step 2: Verify the full feature works end-to-end with a quick smoke test** - -```bash -uv run python -c " -import numpy as np -from orcapod.extension_types.numpy_type import LogicalNumpyArray - -lt = LogicalNumpyArray() - -# 1-D -a = np.array([1.0, 2.0, 3.0], dtype=np.float64) -assert np.array_equal(lt.storage_to_python(lt.python_to_storage(a)), a) - -# 2-D -b = np.array([[1, 2], [3, 4]], dtype=np.int32) -assert np.array_equal(lt.storage_to_python(lt.python_to_storage(b)), b) - -# Structured -c = np.array([(1.0, 2)], dtype=[('x', np.float64), ('y', np.int32)]) -r = lt.storage_to_python(lt.python_to_storage(c)) -assert np.array_equal(r['x'], c['x']) -assert np.array_equal(r['y'], c['y']) - -# Object-dtype rejection -try: - lt.python_to_storage(np.array([1, 'a'], dtype=object)) - assert False, 'Should have raised' -except ValueError as e: - assert 'object-dtype' in str(e) - -print('All smoke tests passed.') -" -``` - -Expected: `All smoke tests passed.` - -- [ ] **Step 3: Push the branch** - -```bash -git push -u origin eywalker/itl-460-support-numpyndarray-as-a-native-value-type -``` diff --git a/superpowers/plans/2026-07-01-pandas-native-types.md b/superpowers/plans/2026-07-01-pandas-native-types.md deleted file mode 100644 index 232851a0a..000000000 --- a/superpowers/plans/2026-07-01-pandas-native-types.md +++ /dev/null @@ -1,976 +0,0 @@ -# pandas.DataFrame and pandas.Series Native Types Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `pd.DataFrame` and `pd.Series` as native orcapod value types — serialised as Arrow IPC stream bytes in `large_binary` extension columns — so the `UniversalTypeConverter` no longer raises `ValueError: Unsupported Python type` for these types. - -**Architecture:** Each type gets a `LogicalXxx(BaseLogicalType)` class in a new `pandas_type.py` module, following the exact structure of `numpy_type.py`. Storage uses Arrow IPC stream bytes (`preserve_index=True`) stored as `large_binary`. Content hashing uses SHA-256 of those same IPC bytes, returned directly as `ContentHash`. Both types and handlers are registered in `contexts/data/v0.1.json`. - -**Tech Stack:** `pandas>=2.2.3` (already a first-class dependency), `pyarrow` (already a dependency), `pyarrow.ipc` for IPC stream serialisation, `pytest`, `uv run` for all commands. - ---- - -## File Map - -| Action | Path | Responsibility | -|--------|------|----------------| -| **Create** | `src/orcapod/extension_types/pandas_type.py` | `LogicalPandasDataFrame` + `LogicalPandasSeries` logical types | -| **Modify** | `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Add `PandasDataFrameHandler` + `PandasSeriesHandler` | -| **Modify** | `src/orcapod/contexts/data/v0.1.json` | Register both logical types + both handlers | -| **Create** | `tests/test_extension_types/test_pandas_type.py` | Unit tests for both logical types | -| **Create** | `tests/test_hashing/test_pandas_handlers.py` | Unit tests for both hash handlers | -| **Modify** | `tests/test_extension_types/test_roundtrips.py` | End-to-end integration tests | - ---- - -## Task 1: `LogicalPandasDataFrame` — class shell + protocol tests - -**Files:** -- Create: `src/orcapod/extension_types/pandas_type.py` -- Create: `tests/test_extension_types/test_pandas_type.py` - -- [ ] **Step 1.1: Write failing protocol tests** - -Create `tests/test_extension_types/test_pandas_type.py`: - -```python -"""Tests for LogicalPandasDataFrame and LogicalPandasSeries.""" -from __future__ import annotations - -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest - -from orcapod.extension_types.protocols import LogicalTypeProtocol - - -class TestLogicalPandasDataFrameProtocol: - def test_isinstance_logical_type(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - assert isinstance(LogicalPandasDataFrame(), LogicalTypeProtocol) - - def test_logical_type_name(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - assert LogicalPandasDataFrame().logical_type_name == "pandas.dataframe" - - def test_python_type(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - assert LogicalPandasDataFrame().python_type is pd.DataFrame - - def test_arrow_ext_name(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - assert LogicalPandasDataFrame().get_arrow_extension_type().extension_name == "pandas.dataframe" - - def test_arrow_ext_storage_type(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - assert LogicalPandasDataFrame().get_arrow_extension_type().storage_type == pa.large_binary() - - def test_arrow_ext_is_cached(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - lt = LogicalPandasDataFrame() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - def test_polars_ext_is_cached(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - lt = LogicalPandasDataFrame() - assert lt.get_polars_extension_type() is lt.get_polars_extension_type() -``` - -- [ ] **Step 1.2: Run to verify failure** - -```bash -uv run pytest tests/test_extension_types/test_pandas_type.py::TestLogicalPandasDataFrameProtocol -v -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.extension_types.pandas_type'` - -- [ ] **Step 1.3: Create `pandas_type.py` with the class shell** - -Create `src/orcapod/extension_types/pandas_type.py`: - -```python -"""``pandas.DataFrame`` and ``pandas.Series`` logical types for orcapod. - -``LogicalPandasDataFrame`` maps ``pd.DataFrame`` <-> Arrow ``large_binary`` using -Arrow IPC stream format as the storage envelope, with ``preserve_index=True`` so -that all index kinds (RangeIndex, named index, MultiIndex) round-trip losslessly. - -``LogicalPandasSeries`` maps ``pd.Series`` <-> Arrow ``large_binary`` by wrapping -the Series as a single-column DataFrame before applying the same IPC path. The -Series name and index are both preserved. An unnamed Series (``name=None``) uses -the sentinel column name ``"__orcapod:unnamed__"`` to distinguish it from a Series -genuinely named ``"__orcapod:unnamed__"`` (the latter would be a bug in user code). -""" - -from __future__ import annotations - -import io -from typing import TYPE_CHECKING, Any - -import pandas as pd -import polars as pl -import pyarrow as pa -import pyarrow.ipc - -from orcapod.extension_types.base_logical_type import BaseLogicalType -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type - -if TYPE_CHECKING: - from orcapod.extension_types.protocols import TypeConverterProtocol - - -_SERIES_UNNAMED_SENTINEL = "__orcapod:unnamed__" - - -def _table_to_ipc_bytes(table: pa.Table) -> bytes: - """Serialise an Arrow Table to IPC stream bytes. - - Args: - table: The Arrow Table to serialise. - - Returns: - Raw bytes in Arrow IPC stream format. - """ - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - return buf.getvalue() - - -def _ipc_bytes_to_table(data: Any) -> pa.Table: - """Deserialise IPC stream bytes to an Arrow Table. - - Args: - data: Raw bytes (or buffer) containing an Arrow IPC stream. - - Returns: - The deserialised Arrow Table. - """ - buf = io.BytesIO(bytes(data)) - return pa.ipc.open_stream(buf).read_all() - - -class LogicalPandasDataFrame(BaseLogicalType): - """Logical type for ``pd.DataFrame``. - - Stores DataFrames as Arrow ``large_binary`` using Arrow IPC stream format. - ``preserve_index=True`` ensures that all index kinds — default ``RangeIndex``, - named indices, and ``MultiIndex`` — are round-tripped losslessly. - - Any DataFrame whose columns are Arrow-serialisable (numeric, string/object, - nullable integer, boolean, datetime, categorical) is supported. Columns with - truly mixed Python types that Arrow cannot infer raise ``ValueError``. - - The extension name ``"pandas.dataframe"`` is library-qualified, following the - ``"numpy.ndarray"`` precedent. - - Example: - >>> lt = LogicalPandasDataFrame() - >>> df = pd.DataFrame({"x": [1.0, 2.0]}, index=pd.Index([10, 20], name="id")) - >>> recovered = lt.storage_to_python(lt.python_to_storage(df)) - >>> df.equals(recovered) - True - """ - - _arrow_ext_class = make_arrow_extension_type("pandas.dataframe", pa.large_binary()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("pandas.dataframe", pa.large_binary()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "pandas.dataframe" - python_type: type = pd.DataFrame - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``pd.DataFrame``. - - Returns: - A ``pa.ExtensionType`` instance with extension name - ``"pandas.dataframe"`` and storage type ``pa.large_binary()``. - """ - if LogicalPandasDataFrame._arrow_ext is None: - LogicalPandasDataFrame._arrow_ext = LogicalPandasDataFrame._arrow_ext_class() - return LogicalPandasDataFrame._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``pd.DataFrame``. - - Returns: - A ``pl.BaseExtension`` instance registered under ``"pandas.dataframe"``. - """ - if LogicalPandasDataFrame._polars_ext is None: - LogicalPandasDataFrame._polars_ext = LogicalPandasDataFrame._polars_ext_class() - return LogicalPandasDataFrame._polars_ext - - def python_to_storage(self, value: Any, converter: TypeConverterProtocol | None = None) -> bytes: - """Serialise a ``pd.DataFrame`` to Arrow IPC stream bytes. - - Args: - value: A ``pd.DataFrame`` instance. All columns must be - Arrow-serialisable (numeric, string, nullable int, bool, - datetime, categorical). Columns with mixed non-Arrow types raise - ``ValueError``. - converter: Ignored. Present for protocol conformance. - - Returns: - Raw bytes in Arrow IPC stream format encoding the DataFrame's - schema, index, and data. - - Raises: - ValueError: If any column cannot be converted to an Arrow type. - """ - try: - table = pa.Table.from_pandas(value, preserve_index=True) - except pa.lib.ArrowInvalid as exc: - raise ValueError( - f"LogicalPandasDataFrame: cannot convert DataFrame to Arrow. " - f"Check for columns with mixed or non-Arrow-serialisable types. " - f"Original error: {exc}" - ) from exc - return _table_to_ipc_bytes(table) - - def storage_to_python(self, storage_value: Any, converter: TypeConverterProtocol | None = None) -> pd.DataFrame: - """Reconstruct a ``pd.DataFrame`` from Arrow IPC stream bytes. - - Args: - storage_value: Raw bytes as stored in Arrow ``large_binary``. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``pd.DataFrame`` with columns, dtypes, and index as originally stored. - """ - return _ipc_bytes_to_table(storage_value).to_pandas() - - -class LogicalPandasSeries(BaseLogicalType): - """Logical type for ``pd.Series``. - - Stores Series as Arrow ``large_binary`` using Arrow IPC stream format. - The Series is wrapped as a single-column DataFrame before serialisation, - which preserves the Series name and index losslessly. - - An unnamed Series (``name=None``) uses the internal sentinel column name - ``"__orcapod:unnamed__"`` to distinguish it from a Series named with an - empty string. - - The extension name ``"pandas.series"`` is library-qualified. - - Example: - >>> lt = LogicalPandasSeries() - >>> s = pd.Series([1.0, 2.0, 3.0], name="metric") - >>> recovered = lt.storage_to_python(lt.python_to_storage(s)) - >>> s.equals(recovered) - True - """ - - _arrow_ext_class = make_arrow_extension_type("pandas.series", pa.large_binary()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("pandas.series", pa.large_binary()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "pandas.series" - python_type: type = pd.Series - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``pd.Series``. - - Returns: - A ``pa.ExtensionType`` instance with extension name - ``"pandas.series"`` and storage type ``pa.large_binary()``. - """ - if LogicalPandasSeries._arrow_ext is None: - LogicalPandasSeries._arrow_ext = LogicalPandasSeries._arrow_ext_class() - return LogicalPandasSeries._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``pd.Series``. - - Returns: - A ``pl.BaseExtension`` instance registered under ``"pandas.series"``. - """ - if LogicalPandasSeries._polars_ext is None: - LogicalPandasSeries._polars_ext = LogicalPandasSeries._polars_ext_class() - return LogicalPandasSeries._polars_ext - - def python_to_storage(self, value: Any, converter: TypeConverterProtocol | None = None) -> bytes: - """Serialise a ``pd.Series`` to Arrow IPC stream bytes. - - Args: - value: A ``pd.Series`` instance. The series name and index are - preserved. An unnamed series (``name=None``) is stored under - the sentinel column name ``"__orcapod:unnamed__"``. - converter: Ignored. Present for protocol conformance. - - Returns: - Raw bytes in Arrow IPC stream format. - - Raises: - ValueError: If the Series values cannot be converted to an Arrow type. - """ - col_name = value.name if value.name is not None else _SERIES_UNNAMED_SENTINEL - df = value.to_frame(name=col_name) - try: - table = pa.Table.from_pandas(df, preserve_index=True) - except pa.lib.ArrowInvalid as exc: - raise ValueError( - f"LogicalPandasSeries: cannot convert Series to Arrow. " - f"Check for non-Arrow-serialisable values. " - f"Original error: {exc}" - ) from exc - return _table_to_ipc_bytes(table) - - def storage_to_python(self, storage_value: Any, converter: TypeConverterProtocol | None = None) -> pd.Series: - """Reconstruct a ``pd.Series`` from Arrow IPC stream bytes. - - Args: - storage_value: Raw bytes as stored in Arrow ``large_binary``. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``pd.Series`` with its original name and index restored. - """ - df = _ipc_bytes_to_table(storage_value).to_pandas() - series = df.iloc[:, 0] - if series.name == _SERIES_UNNAMED_SENTINEL: - series = series.rename(None) - return series -``` - -- [ ] **Step 1.4: Run protocol tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_pandas_type.py::TestLogicalPandasDataFrameProtocol -v -``` - -Expected: 7 tests PASS - -- [ ] **Step 1.5: Commit** - -```bash -git add src/orcapod/extension_types/pandas_type.py tests/test_extension_types/test_pandas_type.py -git commit -m "feat(extension_types): add LogicalPandasDataFrame class shell with protocol tests" -``` - ---- - -## Task 2: `LogicalPandasDataFrame` — round-trip tests - -**Files:** -- Modify: `tests/test_extension_types/test_pandas_type.py` (add classes) -- (Implementation already complete in Task 1's class shell) - -- [ ] **Step 2.1: Add storage and round-trip tests** - -Append to `tests/test_extension_types/test_pandas_type.py`: - -```python -class TestLogicalPandasDataFrameStorage: - def test_python_to_storage_returns_bytes(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - df = pd.DataFrame({"a": [1.0, 2.0], "b": [3, 4]}) - result = LogicalPandasDataFrame().python_to_storage(df) - assert isinstance(result, bytes) - - def test_non_arrow_column_raises_value_error(self): - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - - class _Opaque: - pass - - df = pd.DataFrame({"bad": [_Opaque(), _Opaque()]}) - with pytest.raises(ValueError, match="cannot convert"): - LogicalPandasDataFrame().python_to_storage(df) - - -class TestLogicalPandasDataFrameRoundTrip: - def _rt(self, df: pd.DataFrame) -> pd.DataFrame: - from orcapod.extension_types.pandas_type import LogicalPandasDataFrame - lt = LogicalPandasDataFrame() - return lt.storage_to_python(lt.python_to_storage(df)) - - def test_default_range_index(self): - df = pd.DataFrame({"x": [1.0, 2.0, 3.0], "y": [4, 5, 6]}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_named_index(self): - df = pd.DataFrame({"x": [10, 20]}, index=pd.Index([100, 200], name="id")) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_multiindex(self): - idx = pd.MultiIndex.from_tuples([("a", 1), ("b", 2)], names=["letter", "num"]) - df = pd.DataFrame({"val": [0.5, 1.5]}, index=idx) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_float64_column(self): - df = pd.DataFrame({"f": np.array([1.1, 2.2, 3.3], dtype=np.float64)}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_int32_column(self): - df = pd.DataFrame({"i": np.array([1, 2, 3], dtype=np.int32)}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_uint8_column(self): - df = pd.DataFrame({"u": np.array([0, 128, 255], dtype=np.uint8)}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_string_object_column(self): - df = pd.DataFrame({"s": ["hello", "world", None]}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_nullable_int_column(self): - df = pd.DataFrame({"n": pd.array([1, 2, None], dtype=pd.Int64Dtype())}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_boolean_column(self): - df = pd.DataFrame({"b": [True, False, True]}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_datetime_column(self): - df = pd.DataFrame({"t": pd.to_datetime(["2024-01-01", "2024-06-01"])}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_categorical_column(self): - df = pd.DataFrame({"c": pd.Categorical(["a", "b", "a"])}) - pd.testing.assert_frame_equal(self._rt(df), df) - - def test_empty_dataframe(self): - df = pd.DataFrame({"x": pd.Series([], dtype=float)}) - pd.testing.assert_frame_equal(self._rt(df), df) -``` - -- [ ] **Step 2.2: Run to verify all pass** - -```bash -uv run pytest tests/test_extension_types/test_pandas_type.py::TestLogicalPandasDataFrameStorage tests/test_extension_types/test_pandas_type.py::TestLogicalPandasDataFrameRoundTrip -v -``` - -Expected: all tests PASS (implementation was already complete in Task 1) - -- [ ] **Step 2.3: Commit** - -```bash -git add tests/test_extension_types/test_pandas_type.py -git commit -m "test(extension_types): add round-trip tests for LogicalPandasDataFrame" -``` - ---- - -## Task 3: `LogicalPandasSeries` — protocol + round-trip tests - -**Files:** -- Modify: `tests/test_extension_types/test_pandas_type.py` (add classes) -- (Implementation already complete in Task 1's class shell) - -- [ ] **Step 3.1: Add `LogicalPandasSeries` tests** - -Append to `tests/test_extension_types/test_pandas_type.py`: - -```python -class TestLogicalPandasSeriesProtocol: - def test_isinstance_logical_type(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - assert isinstance(LogicalPandasSeries(), LogicalTypeProtocol) - - def test_logical_type_name(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - assert LogicalPandasSeries().logical_type_name == "pandas.series" - - def test_python_type(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - assert LogicalPandasSeries().python_type is pd.Series - - def test_arrow_ext_name(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - assert LogicalPandasSeries().get_arrow_extension_type().extension_name == "pandas.series" - - def test_arrow_ext_storage_type(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - assert LogicalPandasSeries().get_arrow_extension_type().storage_type == pa.large_binary() - - def test_arrow_ext_is_cached(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - lt = LogicalPandasSeries() - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - def test_polars_ext_is_cached(self): - from orcapod.extension_types.pandas_type import LogicalPandasSeries - lt = LogicalPandasSeries() - assert lt.get_polars_extension_type() is lt.get_polars_extension_type() - - -class TestLogicalPandasSeriesRoundTrip: - def _rt(self, s: pd.Series) -> pd.Series: - from orcapod.extension_types.pandas_type import LogicalPandasSeries - lt = LogicalPandasSeries() - return lt.storage_to_python(lt.python_to_storage(s)) - - def test_unnamed_series_name_is_none(self): - s = pd.Series([1.0, 2.0, 3.0]) - result = self._rt(s) - pd.testing.assert_series_equal(result, s) - assert result.name is None - - def test_named_series(self): - s = pd.Series([10, 20, 30], name="count") - result = self._rt(s) - pd.testing.assert_series_equal(result, s) - assert result.name == "count" - - def test_named_index_preserved(self): - s = pd.Series([1.0, 2.0], index=pd.Index([10, 20], name="row_id"), name="val") - pd.testing.assert_series_equal(self._rt(s), s) - - def test_float64_values(self): - s = pd.Series(np.array([1.1, 2.2], dtype=np.float64), name="f") - pd.testing.assert_series_equal(self._rt(s), s) - - def test_int32_values(self): - s = pd.Series(np.array([1, 2, 3], dtype=np.int32), name="i") - pd.testing.assert_series_equal(self._rt(s), s) - - def test_string_values(self): - s = pd.Series(["a", "b", None], name="words") - pd.testing.assert_series_equal(self._rt(s), s) - - def test_bool_values(self): - s = pd.Series([True, False, True], name="flags") - pd.testing.assert_series_equal(self._rt(s), s) - - def test_datetime_values(self): - s = pd.Series(pd.to_datetime(["2024-01-01", "2024-06-01"]), name="ts") - pd.testing.assert_series_equal(self._rt(s), s) - - def test_empty_series(self): - s = pd.Series([], dtype=float, name="empty") - pd.testing.assert_series_equal(self._rt(s), s) -``` - -- [ ] **Step 3.2: Run all pandas type tests** - -```bash -uv run pytest tests/test_extension_types/test_pandas_type.py -v -``` - -Expected: all tests PASS - -- [ ] **Step 3.3: Commit** - -```bash -git add tests/test_extension_types/test_pandas_type.py -git commit -m "test(extension_types): add protocol and round-trip tests for LogicalPandasSeries" -``` - ---- - -## Task 4: Hashing handlers - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Create: `tests/test_hashing/test_pandas_handlers.py` - -- [ ] **Step 4.1: Write failing handler tests** - -Create `tests/test_hashing/test_pandas_handlers.py`: - -```python -"""Tests for PandasDataFrameHandler and PandasSeriesHandler.""" -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from orcapod.types import ContentHash - - -class TestPandasDataFrameHandler: - def test_returns_content_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - df = pd.DataFrame({"a": [1.0, 2.0], "b": [3, 4]}) - result = PandasDataFrameHandler().handle(df, hasher=None) - assert isinstance(result, ContentHash) - assert result.method == "sha256" - - def test_same_content_same_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - handler = PandasDataFrameHandler() - df = pd.DataFrame({"x": [1.0, 2.0, 3.0]}) - assert handler.handle(df, None) == handler.handle(df.copy(), None) - - def test_different_data_different_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - handler = PandasDataFrameHandler() - df1 = pd.DataFrame({"x": [1.0, 2.0]}) - df2 = pd.DataFrame({"x": [3.0, 4.0]}) - assert handler.handle(df1, None).digest != handler.handle(df2, None).digest - - def test_different_index_different_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - handler = PandasDataFrameHandler() - df1 = pd.DataFrame({"x": [1, 2]}, index=[0, 1]) - df2 = pd.DataFrame({"x": [1, 2]}, index=[10, 20]) - assert handler.handle(df1, None).digest != handler.handle(df2, None).digest - - def test_different_column_names_different_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - handler = PandasDataFrameHandler() - df1 = pd.DataFrame({"a": [1, 2]}) - df2 = pd.DataFrame({"b": [1, 2]}) - assert handler.handle(df1, None).digest != handler.handle(df2, None).digest - - def test_wrong_type_raises_type_error(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - with pytest.raises(TypeError, match="PandasDataFrameHandler"): - PandasDataFrameHandler().handle("not a dataframe", hasher=None) - - def test_non_arrow_column_raises_value_error(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasDataFrameHandler - - class _Opaque: - pass - - df = pd.DataFrame({"bad": [_Opaque(), _Opaque()]}) - with pytest.raises(ValueError, match="cannot convert"): - PandasDataFrameHandler().handle(df, hasher=None) - - -class TestPandasSeriesHandler: - def test_returns_content_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasSeriesHandler - s = pd.Series([1.0, 2.0, 3.0], name="x") - result = PandasSeriesHandler().handle(s, hasher=None) - assert isinstance(result, ContentHash) - assert result.method == "sha256" - - def test_same_content_same_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasSeriesHandler - handler = PandasSeriesHandler() - s = pd.Series([1.0, 2.0, 3.0]) - assert handler.handle(s, None) == handler.handle(s.copy(), None) - - def test_different_data_different_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasSeriesHandler - handler = PandasSeriesHandler() - s1 = pd.Series([1.0, 2.0]) - s2 = pd.Series([3.0, 4.0]) - assert handler.handle(s1, None).digest != handler.handle(s2, None).digest - - def test_unnamed_and_named_different_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasSeriesHandler - handler = PandasSeriesHandler() - s1 = pd.Series([1.0, 2.0]) # name=None → stored as sentinel - s2 = pd.Series([1.0, 2.0], name="x") # name="x" - assert handler.handle(s1, None).digest != handler.handle(s2, None).digest - - def test_different_index_different_hash(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasSeriesHandler - handler = PandasSeriesHandler() - s1 = pd.Series([1, 2], index=[0, 1], name="v") - s2 = pd.Series([1, 2], index=[10, 20], name="v") - assert handler.handle(s1, None).digest != handler.handle(s2, None).digest - - def test_wrong_type_raises_type_error(self): - from orcapod.hashing.semantic_hashing.builtin_handlers import PandasSeriesHandler - with pytest.raises(TypeError, match="PandasSeriesHandler"): - PandasSeriesHandler().handle(42, hasher=None) -``` - -- [ ] **Step 4.2: Run to verify failure** - -```bash -uv run pytest tests/test_hashing/test_pandas_handlers.py -v -``` - -Expected: `ImportError` — `PandasDataFrameHandler` / `PandasSeriesHandler` do not exist yet - -- [ ] **Step 4.3: Add handlers to `builtin_handlers.py`** - -Append the following two classes to the end of `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` (after the existing `NumpyArrayHandler` class): - -```python -class PandasDataFrameHandler: - """Hasher for ``pd.DataFrame`` — content hash via SHA-256 of Arrow IPC stream bytes. - - Serialises the DataFrame to Arrow IPC stream format (with ``preserve_index=True``) - and returns a ``ContentHash`` produced by SHA-256 of those bytes. The serialisation - path is identical to ``LogicalPandasDataFrame.python_to_storage``, so the hash - input and storage representation are always consistent. - - Returning ``ContentHash`` directly (rather than the raw bytes) avoids the - hex-expansion and JSON-serialisation overhead that the semantic hasher would apply - to ``bytes`` returns — important for large DataFrames. - """ - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: - """Return a SHA-256 ``ContentHash`` of the IPC bytes for ``obj``. - - Args: - obj: A ``pd.DataFrame`` instance. - hasher: Ignored. Present for protocol conformance. - - Returns: - A ``ContentHash`` with ``method="sha256"`` and digest equal to the - SHA-256 of the DataFrame's Arrow IPC stream bytes. - - Raises: - TypeError: If ``obj`` is not a ``pd.DataFrame``. - ValueError: If any column cannot be converted to an Arrow type. - """ - import hashlib - import io - import pandas as pd - import pyarrow as pa - import pyarrow.ipc - - if not isinstance(obj, pd.DataFrame): - raise TypeError( - f"PandasDataFrameHandler: expected pd.DataFrame, got {type(obj)!r}" - ) - try: - table = pa.Table.from_pandas(obj, preserve_index=True) - except pa.lib.ArrowInvalid as exc: - raise ValueError( - f"PandasDataFrameHandler: cannot convert DataFrame to Arrow. " - f"Check for columns with mixed or non-Arrow-serialisable types. " - f"Original error: {exc}" - ) from exc - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - return ContentHash(method="sha256", digest=hashlib.sha256(buf.getvalue()).digest()) - - -class PandasSeriesHandler: - """Hasher for ``pd.Series`` — content hash via SHA-256 of Arrow IPC stream bytes. - - Wraps the Series as a single-column DataFrame (using the sentinel column name - ``"__orcapod:unnamed__"`` for an unnamed Series) and applies the same IPC - serialisation path as ``PandasDataFrameHandler``. The hash input is always - consistent with what ``LogicalPandasSeries.python_to_storage`` stores. - """ - - _UNNAMED_SENTINEL = "__orcapod:unnamed__" - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: - """Return a SHA-256 ``ContentHash`` of the IPC bytes for ``obj``. - - Args: - obj: A ``pd.Series`` instance. - hasher: Ignored. Present for protocol conformance. - - Returns: - A ``ContentHash`` with ``method="sha256"`` and digest equal to the - SHA-256 of the Series' Arrow IPC stream bytes. - - Raises: - TypeError: If ``obj`` is not a ``pd.Series``. - ValueError: If the Series values cannot be converted to an Arrow type. - """ - import hashlib - import io - import pandas as pd - import pyarrow as pa - import pyarrow.ipc - - if not isinstance(obj, pd.Series): - raise TypeError( - f"PandasSeriesHandler: expected pd.Series, got {type(obj)!r}" - ) - col_name = obj.name if obj.name is not None else self._UNNAMED_SENTINEL - df = obj.to_frame(name=col_name) - try: - table = pa.Table.from_pandas(df, preserve_index=True) - except pa.lib.ArrowInvalid as exc: - raise ValueError( - f"PandasSeriesHandler: cannot convert Series to Arrow. " - f"Check for non-Arrow-serialisable values. " - f"Original error: {exc}" - ) from exc - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - return ContentHash(method="sha256", digest=hashlib.sha256(buf.getvalue()).digest()) -``` - -- [ ] **Step 4.4: Run handler tests** - -```bash -uv run pytest tests/test_hashing/test_pandas_handlers.py -v -``` - -Expected: all tests PASS - -- [ ] **Step 4.5: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py tests/test_hashing/test_pandas_handlers.py -git commit -m "feat(hashing): add PandasDataFrameHandler and PandasSeriesHandler" -``` - ---- - -## Task 5: Registration in `v0.1.json` + integration tests - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` -- Modify: `tests/test_extension_types/test_roundtrips.py` - -- [ ] **Step 5.1: Write failing integration tests** - -Append to `tests/test_extension_types/test_roundtrips.py`: - -```python -def test_builtin_dataframe_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """pd.DataFrame round-trips through storage with extension name ``pandas.dataframe``.""" - import pandas as pd - df = pd.DataFrame( - {"x": [1.0, 2.0, 3.0], "label": ["a", "b", "c"]}, - index=pd.Index([10, 20, 30], name="row_id"), - ) - result, read_converter = _write_and_read( - {"frame": pd.DataFrame}, - [{"frame": df}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("frame") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'frame', got plain type {field.type!r}" - ) - assert field.type.extension_name == "pandas.dataframe" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - recovered = rows[0]["frame"] - assert isinstance(recovered, pd.DataFrame) - pd.testing.assert_frame_equal(recovered, df) - - -def test_builtin_series_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """pd.Series round-trips through storage with extension name ``pandas.series``.""" - import pandas as pd - s = pd.Series([10.0, 20.0, 30.0], name="metric", index=pd.Index([1, 2, 3], name="id")) - result, read_converter = _write_and_read( - {"series": pd.Series}, - [{"series": s}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("series") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on field 'series', got plain type {field.type!r}" - ) - assert field.type.extension_name == "pandas.series" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - recovered = rows[0]["series"] - assert isinstance(recovered, pd.Series) - pd.testing.assert_series_equal(recovered, s) -``` - -- [ ] **Step 5.2: Run to verify failure** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py::test_builtin_dataframe_round_trip tests/test_extension_types/test_roundtrips.py::test_builtin_series_round_trip -v -``` - -Expected: FAIL with `ValueError: Unsupported Python type` (types not yet registered in context) - -- [ ] **Step 5.3: Register both logical types in `v0.1.json`** - -In `src/orcapod/contexts/data/v0.1.json`, find the `logical_types` array and append after the numpy entry: - -```json -{ - "_class": "orcapod.extension_types.numpy_type.LogicalNumpyArray", - "_config": {} -}, -{ - "_class": "orcapod.extension_types.pandas_type.LogicalPandasDataFrame", - "_config": {} -}, -{ - "_class": "orcapod.extension_types.pandas_type.LogicalPandasSeries", - "_config": {} -} -``` - -- [ ] **Step 5.4: Register both handlers in `v0.1.json`** - -In `src/orcapod/contexts/data/v0.1.json`, find the `python_type_handler_registry` handlers array and append after the numpy handler entry: - -```json -[{"_type": "numpy.ndarray"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.NumpyArrayHandler", "_config": {}}], -[{"_type": "pandas.core.frame.DataFrame"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PandasDataFrameHandler", "_config": {}}], -[{"_type": "pandas.core.series.Series"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PandasSeriesHandler", "_config": {}}] -``` - -- [ ] **Step 5.5: Update the changelog in `v0.1.json`** - -Append to the `changelog` array in `v0.1.json`: - -```json -"Added pandas.DataFrame and pandas.Series as native value types via LogicalPandasDataFrame and LogicalPandasSeries (Arrow IPC / large_binary), with index preservation and PandasDataFrameHandler / PandasSeriesHandler for content hashing (PLT-1869)" -``` - -- [ ] **Step 5.6: Run integration tests** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py::test_builtin_dataframe_round_trip tests/test_extension_types/test_roundtrips.py::test_builtin_series_round_trip -v -``` - -Expected: all variants (parquet + delta backends) PASS - -- [ ] **Step 5.7: Run the full test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests pass, no regressions - -- [ ] **Step 5.8: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json tests/test_extension_types/test_roundtrips.py -git commit -m "feat(contexts): register LogicalPandasDataFrame/Series and pandas hashing handlers in v0.1 context (PLT-1869)" -``` - ---- - -## Final Step: Push and open PR - -- [ ] **Push the branch** - -```bash -git push -u origin eywalker/plt-1869-orcapod-support-pandasdataframe-as-a-native-value-type -``` - -- [ ] **Open PR against `main`** - -```bash -gh pr create \ - --title "feat(extension_types): add pandas.DataFrame and pandas.Series as native value types" \ - --body "$(cat <<'EOF' -## Summary - -- Adds `LogicalPandasDataFrame` and `LogicalPandasSeries` to `extension_types/pandas_type.py`, mapping `pd.DataFrame` / `pd.Series` to Arrow `large_binary` via Arrow IPC stream bytes with `preserve_index=True` -- Adds `PandasDataFrameHandler` and `PandasSeriesHandler` in `builtin_handlers.py` for SHA-256 content hashing -- Registers both types and handlers in `contexts/data/v0.1.json` -- Full test coverage: protocol conformance, round-trips (12 DataFrame cases, 8 Series cases), hashing determinism, error paths, end-to-end integration over Parquet and Delta backends - -Closes PLT-1869 - -## Test plan -- [ ] `uv run pytest tests/test_extension_types/test_pandas_type.py -v` — all pass -- [ ] `uv run pytest tests/test_hashing/test_pandas_handlers.py -v` — all pass -- [ ] `uv run pytest tests/test_extension_types/test_roundtrips.py -v` — all pass (parquet + delta) -- [ ] `uv run pytest tests/ -x -q` — no regressions - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF - )" -``` diff --git a/superpowers/plans/2026-07-02-itl-472-file-hash-caching.md b/superpowers/plans/2026-07-02-itl-472-file-hash-caching.md deleted file mode 100644 index e292ca565..000000000 --- a/superpowers/plans/2026-07-02-itl-472-file-hash-caching.md +++ /dev/null @@ -1,1205 +0,0 @@ -# File Hash Caching Implementation Plan (ITL-472) - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `CacherProtocol[K, V]`, `FileHashKey`, `SqliteHashCacher`, and `enable_file_hash_caching()` so Orcapod can cache file hashes in SQLite and reduce repeat hashing to a sub-millisecond lookup. - -**Architecture:** `CacherProtocol[K, V]` is a generic two-method protocol (`get`/`put`). `CachedFileHasher` wraps any `FileContentHasherProtocol` with any `CacherProtocol[FileHashKey, ContentHash]`. `SqliteHashCacher` is the production backend using stdlib `sqlite3` with WAL mode and a purpose-built schema. `enable_file_hash_caching()` wires caching into the default context by re-registering the live `FileHandler` with a `CachedFileHasher`. - -**Tech Stack:** Python stdlib `sqlite3`, `upath.UPath`, `threading.local`, `pytest`, `dataclasses`. - ---- - -## File Map - -| Action | File | What changes | -|---|---|---| -| Modify | `src/orcapod/protocols/hashing_protocols.py` | Add `CacherProtocol[K, V]` | -| Modify | `src/orcapod/hashing/file_hashers.py` | Add `FileHashKey`; rename `BasicFileHasher` → `FileHasher`; update `CachedFileHasher` to use `CacherProtocol` + UPath | -| **Create** | `src/orcapod/hashing/hash_cachers.py` | `InMemoryHashCacher`, `SqliteHashCacher` | -| Modify | `src/orcapod/hashing/__init__.py` | Add new exports, remove `BasicFileHasher` | -| Modify | `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Update fallback import `BasicFileHasher` → `FileHasher` | -| Modify | `src/orcapod/contexts/data/v0.1.json` | Update `_class` to `FileHasher` | -| Modify | `src/orcapod/contexts/__init__.py` | Add `enable_file_hash_caching()` | -| Modify | `tests/test_hashing/test_file_hashers.py` | Rename `BasicFileHasher` → `FileHasher`; update `CachedFileHasher` tests | -| **Create** | `tests/test_hashing/test_hash_cachers.py` | `InMemoryHashCacher`, `SqliteHashCacher`, `enable_file_hash_caching()` tests | -| **Create** | `bench/bench_file_hasher_cache.py` | Timing benchmark | - ---- - -## Task 1: Add `CacherProtocol[K, V]` - -**Files:** -- Modify: `src/orcapod/protocols/hashing_protocols.py` - -- [ ] **Step 1: Add TypeVars and `CacherProtocol` to `hashing_protocols.py`** - -Open `src/orcapod/protocols/hashing_protocols.py`. At the top of the file, `from __future__ import annotations` may or may not be present — check first. Add the TypeVars directly before the `StringCacherProtocol` class, and add `CacherProtocol` right after it: - -```python -# Add near the top imports: -from typing import TypeVar - -# ... existing code ... - -# Add before StringCacherProtocol: -K = TypeVar("K") -V = TypeVar("V") - - -class CacherProtocol(Protocol[K, V]): - """Generic get/put caching protocol. - - A two-method protocol for caches keyed and valued by arbitrary types. - Use typed specializations (e.g. ``CacherProtocol[FileHashKey, ContentHash]``) - to constrain implementations. - - Type Parameters: - K: The cache key type. - V: The cached value type. - """ - - def get(self, key: K) -> V | None: - """Return the cached value for ``key``, or ``None`` on miss.""" - ... - - def put(self, key: K, value: V) -> None: - """Store ``value`` under ``key``.""" - ... -``` - -- [ ] **Step 2: Verify it imports cleanly** - -```bash -cd /home/kurouto/kurouto-jobs/e01df523-d89f-44e2-8417-5a50eb2a506a/orcapod-python -uv run python -c "from orcapod.protocols.hashing_protocols import CacherProtocol; print('ok')" -``` - -Expected output: `ok` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/protocols/hashing_protocols.py -git commit -m "feat(hashing): add generic CacherProtocol[K, V] to hashing_protocols" -``` - ---- - -## Task 2: Add `FileHashKey` and rename `BasicFileHasher` → `FileHasher` - -**Files:** -- Modify: `src/orcapod/hashing/file_hashers.py` -- Modify: `src/orcapod/hashing/__init__.py` -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Modify: `src/orcapod/contexts/data/v0.1.json` -- Modify: `tests/test_hashing/test_file_hashers.py` -- Modify: `tests/test_hashing/test_file_handler.py` - -- [ ] **Step 1: Rewrite `src/orcapod/hashing/file_hashers.py`** - -Replace the entire file contents with: - -```python -import os -from dataclasses import dataclass -from pathlib import Path - -from upath import UPath - -from orcapod.hashing.hash_utils import hash_file -from orcapod.protocols.hashing_protocols import ( - CacherProtocol, - FileContentHasherProtocol, -) -from orcapod.types import ContentHash, PathLike - - -@dataclass(frozen=True) -class FileHashKey: - """Cache lookup key for a file hash. - - The key captures the three file attributes that together identify a - file's content without reading it: its absolute path, last-modified - time in nanoseconds, and byte size. - - Attributes: - path: Absolute, resolved ``UPath``. Any ``PathLike`` input is - normalised to ``UPath`` before constructing this key. - mtime_ns: Last-modified time in nanoseconds (``os.stat().st_mtime_ns``). - size: File size in bytes (``os.stat().st_size``). - """ - - path: UPath - mtime_ns: int - size: int - - -class FileHasher: - """Hash file content using a configurable algorithm. - - Args: - algorithm: Hashing algorithm to use. Defaults to ``"sha256"``. - buffer_size: Read buffer size in bytes. Defaults to 65536. - """ - - def __init__( - self, - algorithm: str = "sha256", - buffer_size: int = 65536, - ): - self.algorithm = algorithm - self.buffer_size = buffer_size - - def hash_file(self, file_path: PathLike) -> ContentHash: - """Hash the file at ``file_path`` and return its ``ContentHash``. - - Args: - file_path: Path to the file to hash. - - Returns: - ContentHash of the file's content. - """ - return hash_file( - file_path, algorithm=self.algorithm, buffer_size=self.buffer_size - ) - - -class CachedFileHasher: - """File hasher that caches results to avoid redundant I/O. - - Wraps any ``FileContentHasherProtocol`` with a - ``CacherProtocol[FileHashKey, ContentHash]``. On each call to - ``hash_file``: - - 1. Resolve the path to an absolute ``UPath`` and stat the file. - 2. Look up ``(path, mtime_ns, size)`` in the cacher. - 3. On hit: return the cached ``ContentHash`` directly. - 4. On miss: delegate to the inner hasher, store the result, return it. - - Both ``FileHasher`` and ``CachedFileHasher`` satisfy - ``FileContentHasherProtocol`` — callers do not need to know which they - have. - - Args: - file_hasher: Inner hasher that performs the actual content hashing. - cacher: Cache backend implementing - ``CacherProtocol[FileHashKey, ContentHash]``. - """ - - def __init__( - self, - file_hasher: FileContentHasherProtocol, - cacher: "CacherProtocol[FileHashKey, ContentHash]", - ) -> None: - self.file_hasher = file_hasher - self.cacher = cacher - - def hash_file(self, file_path: PathLike) -> ContentHash: - """Return the ``ContentHash`` for ``file_path``, using the cache. - - Args: - file_path: Path to the file to hash. - - Returns: - ContentHash of the file's content (from cache or computed). - """ - path = file_path if isinstance(file_path, UPath) else UPath(file_path) - path = path.resolve() - stat = path.stat() - key = FileHashKey(path, stat.st_mtime_ns, stat.st_size) - - hit = self.cacher.get(key) - if hit is not None: - return hit - - result = self.file_hasher.hash_file(file_path) - self.cacher.put(key, result) - return result -``` - -- [ ] **Step 2: Update `src/orcapod/hashing/__init__.py`** - -Find and replace these lines: - -```python -# OLD: -from orcapod.hashing.file_hashers import BasicFileHasher, CachedFileHasher -``` -```python -# NEW: -from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher, FileHashKey -``` - -In the `__all__` list, replace `"BasicFileHasher"` with `"FileHasher"` and add `"FileHashKey"`: - -```python -# OLD: - "BasicFileHasher", - "CachedFileHasher", -``` -```python -# NEW: - "FileHasher", - "FileHashKey", - "CachedFileHasher", -``` - -- [ ] **Step 3: Update `src/orcapod/hashing/semantic_hashing/builtin_handlers.py`** - -Find lines 453–454 (the fallback import inside `register_builtin_python_type_handlers`): - -```python -# OLD: - from orcapod.hashing.file_hashers import BasicFileHasher - file_hasher = BasicFileHasher(algorithm="sha256") -``` -```python -# NEW: - from orcapod.hashing.file_hashers import FileHasher - file_hasher = FileHasher(algorithm="sha256") -``` - -Also update the docstring on line 444 from `BasicFileHasher(sha256)` to `FileHasher(sha256)`. - -- [ ] **Step 4: Update `src/orcapod/contexts/data/v0.1.json`** - -```json -// OLD: -"_class": "orcapod.hashing.file_hashers.BasicFileHasher", -``` -```json -// NEW: -"_class": "orcapod.hashing.file_hashers.FileHasher", -``` - -- [ ] **Step 5: Update `tests/test_hashing/test_file_hashers.py`** - -Replace the import line: -```python -# OLD: -from orcapod.hashing.file_hashers import BasicFileHasher, CachedFileHasher -from orcapod.hashing.string_cachers import InMemoryCacher -``` -```python -# NEW: -from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher, FileHashKey -from orcapod.hashing.hash_cachers import InMemoryHashCacher -``` - -Replace every occurrence of `BasicFileHasher` with `FileHasher` (class name, fixture, docstrings, comments). - -Replace every occurrence of `string_cacher=cacher` with `cacher=cacher` and `InMemoryCacher()` with `InMemoryHashCacher()`. - -Replace the fixture: -```python -# OLD: -@pytest.fixture -def file_hasher(): - return BasicFileHasher(algorithm="sha256") - -@pytest.fixture -def cached_file_hasher(file_hasher): - cacher = InMemoryCacher() - return CachedFileHasher(file_hasher=file_hasher, string_cacher=cacher) -``` -```python -# NEW: -@pytest.fixture -def file_hasher(): - return FileHasher(algorithm="sha256") - -@pytest.fixture -def cached_file_hasher(file_hasher): - cacher = InMemoryHashCacher() - return CachedFileHasher(file_hasher=file_hasher, cacher=cacher) -``` - -Replace `TestBasicFileHasherReturnType` class name with `TestFileHasherReturnType` and update its docstring. - -Replace `test_cache_stores_to_string_format` with a version that inspects the typed cache directly: - -```python -def test_cache_stores_content_hash_directly(sample_file): - """The cacher stores ContentHash objects keyed by FileHashKey.""" - from upath import UPath - inner = FileHasher(algorithm="sha256") - cacher = InMemoryHashCacher() - cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - - result = cached.hash_file(sample_file) - - path = UPath(sample_file).resolve() - stat = path.stat() - key = FileHashKey(path, stat.st_mtime_ns, stat.st_size) - cached_value = cacher.get(key) - - assert cached_value is not None - assert cached_value == result - assert cached_value.method == "sha256" - assert isinstance(cached_value.digest, bytes) -``` - -Update `test_different_algorithms_share_cache_key`: -```python -def test_different_algorithms_share_cache_key(tmp_path): - """Two CachedFileHashers with different algorithms but same path+mtime+size - share the same cache entry — second gets the first's result. - - This is a known limitation: the cache key does not include the algorithm. - """ - f = tmp_path / "file.txt" - f.write_text("test content") - - cacher = InMemoryHashCacher() - - sha_inner = FileHasher(algorithm="sha256") - sha_cached = CachedFileHasher(file_hasher=sha_inner, cacher=cacher) - sha_result = sha_cached.hash_file(f) - - md5_inner = FileHasher(algorithm="md5") - md5_cached = CachedFileHasher(file_hasher=md5_inner, cacher=cacher) - md5_result = md5_cached.hash_file(f) - - assert md5_result.method == "sha256" - assert md5_result.digest == sha_result.digest -``` - -- [ ] **Step 6: Update `tests/test_hashing/test_file_handler.py`** - -Replace: -```python -# OLD: -from orcapod.hashing.file_hashers import BasicFileHasher -``` -```python -# NEW: -from orcapod.hashing.file_hashers import FileHasher -``` - -Replace every `BasicFileHasher(` with `FileHasher(` in that file. - -- [ ] **Step 7: Run the existing test suite (it will fail — `InMemoryHashCacher` not yet defined)** - -```bash -uv run pytest tests/test_hashing/test_file_hashers.py tests/test_hashing/test_file_handler.py -x -q 2>&1 | head -30 -``` - -Expected: fails with `ModuleNotFoundError` or `ImportError` because `InMemoryHashCacher` doesn't exist yet. That is correct — we implement it in Task 3. - -- [ ] **Step 8: Commit the rename (tests broken — that's expected)** - -```bash -git add src/orcapod/hashing/file_hashers.py \ - src/orcapod/hashing/__init__.py \ - src/orcapod/hashing/semantic_hashing/builtin_handlers.py \ - src/orcapod/contexts/data/v0.1.json \ - tests/test_hashing/test_file_hashers.py \ - tests/test_hashing/test_file_handler.py -git commit -m "refactor(hashing): rename BasicFileHasher to FileHasher; add FileHashKey; update CachedFileHasher to CacherProtocol" -``` - ---- - -## Task 3: Implement `InMemoryHashCacher` and `SqliteHashCacher` - -**Files:** -- Create: `src/orcapod/hashing/hash_cachers.py` -- Create: `tests/test_hashing/test_hash_cachers.py` - -- [ ] **Step 1: Write failing tests in `tests/test_hashing/test_hash_cachers.py`** - -Create the file with this content: - -```python -"""Tests for InMemoryHashCacher and SqliteHashCacher.""" - -import sqlite3 -import threading - -import pytest -from upath import UPath - -from orcapod.hashing.file_hashers import FileHashKey -from orcapod.types import ContentHash - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def make_key(path_str: str, mtime_ns: int = 1000, size: int = 100) -> FileHashKey: - return FileHashKey(path=UPath(path_str), mtime_ns=mtime_ns, size=size) - - -def make_hash(method: str = "sha256", digest: bytes = b"\xab" * 32) -> ContentHash: - return ContentHash(method=method, digest=digest) - - -# --------------------------------------------------------------------------- -# InMemoryHashCacher -# --------------------------------------------------------------------------- - - -class TestInMemoryHashCacher: - def test_miss_returns_none(self): - from orcapod.hashing.hash_cachers import InMemoryHashCacher - cacher = InMemoryHashCacher() - assert cacher.get(make_key("/a/b.txt")) is None - - def test_put_then_get_returns_hit(self): - from orcapod.hashing.hash_cachers import InMemoryHashCacher - cacher = InMemoryHashCacher() - key = make_key("/a/b.txt") - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - def test_different_mtime_is_different_key(self): - from orcapod.hashing.hash_cachers import InMemoryHashCacher - cacher = InMemoryHashCacher() - key1 = make_key("/a/b.txt", mtime_ns=1000) - key2 = make_key("/a/b.txt", mtime_ns=2000) - cacher.put(key1, make_hash(digest=b"\xaa" * 32)) - assert cacher.get(key2) is None - - def test_different_size_is_different_key(self): - from orcapod.hashing.hash_cachers import InMemoryHashCacher - cacher = InMemoryHashCacher() - key1 = make_key("/a/b.txt", size=100) - key2 = make_key("/a/b.txt", size=200) - cacher.put(key1, make_hash(digest=b"\xaa" * 32)) - assert cacher.get(key2) is None - - def test_clear_removes_all_entries(self): - from orcapod.hashing.hash_cachers import InMemoryHashCacher - cacher = InMemoryHashCacher() - key = make_key("/a/b.txt") - cacher.put(key, make_hash()) - cacher.clear() - assert cacher.get(key) is None - - -# --------------------------------------------------------------------------- -# SqliteHashCacher -# --------------------------------------------------------------------------- - - -class TestSqliteHashCacher: - def test_miss_returns_none(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - cacher = SqliteHashCacher(tmp_path / "cache.db") - assert cacher.get(make_key("/a/b.txt")) is None - - def test_put_then_get_returns_hit(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - cacher = SqliteHashCacher(tmp_path / "cache.db") - key = make_key("/a/b.txt") - value = make_hash() - cacher.put(key, value) - result = cacher.get(key) - assert result is not None - assert result.method == value.method - assert result.digest == value.digest - - def test_different_mtime_is_different_key(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - cacher = SqliteHashCacher(tmp_path / "cache.db") - key1 = make_key("/a/b.txt", mtime_ns=1000) - key2 = make_key("/a/b.txt", mtime_ns=2000) - cacher.put(key1, make_hash(digest=b"\xaa" * 32)) - assert cacher.get(key2) is None - - def test_different_size_is_different_key(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - cacher = SqliteHashCacher(tmp_path / "cache.db") - key1 = make_key("/a/b.txt", size=100) - key2 = make_key("/a/b.txt", size=200) - cacher.put(key1, make_hash(digest=b"\xaa" * 32)) - assert cacher.get(key2) is None - - def test_persistence_across_instances(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - db = tmp_path / "cache.db" - key = make_key("/a/b.txt") - value = make_hash() - - cacher1 = SqliteHashCacher(db) - cacher1.put(key, value) - cacher1.close() - - cacher2 = SqliteHashCacher(db) - result = cacher2.get(key) - assert result is not None - assert result.method == value.method - assert result.digest == value.digest - - def test_wal_mode_enabled(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - db = tmp_path / "cache.db" - SqliteHashCacher(db) - with sqlite3.connect(db) as conn: - cursor = conn.execute("PRAGMA journal_mode") - mode = cursor.fetchone()[0] - assert mode == "wal" - - def test_clear_empties_table(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - cacher = SqliteHashCacher(tmp_path / "cache.db") - key = make_key("/a/b.txt") - cacher.put(key, make_hash()) - cacher.clear() - assert cacher.get(key) is None - - def test_env_var_path_override(self, tmp_path, monkeypatch): - from orcapod.hashing.hash_cachers import SqliteHashCacher - db = tmp_path / "env_cache.db" - monkeypatch.setenv("ORCAPOD_HASH_CACHE_DB", str(db)) - cacher = SqliteHashCacher() - assert cacher.db_path == db - - def test_context_manager_closes_connection(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - db = tmp_path / "cache.db" - with SqliteHashCacher(db) as cacher: - cacher.put(make_key("/a/b.txt"), make_hash()) - # After __exit__, thread-local conn should be None — verify by - # reopening and checking persistence still works - cacher2 = SqliteHashCacher(db) - assert cacher2.get(make_key("/a/b.txt")) is not None - - def test_idempotent_put(self, tmp_path): - from orcapod.hashing.hash_cachers import SqliteHashCacher - cacher = SqliteHashCacher(tmp_path / "cache.db") - key = make_key("/a/b.txt") - v1 = make_hash(digest=b"\xaa" * 32) - v2 = make_hash(digest=b"\xbb" * 32) - cacher.put(key, v1) - cacher.put(key, v2) # INSERT OR REPLACE - result = cacher.get(key) - assert result.digest == v2.digest -``` - -- [ ] **Step 2: Run tests to verify they fail (module not found)** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py -x -q 2>&1 | head -20 -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.hashing.hash_cachers'` - -- [ ] **Step 3: Create `src/orcapod/hashing/hash_cachers.py`** - -```python -"""File hash cacher implementations. - -Provides ``InMemoryHashCacher`` (testing/ephemeral use) and -``SqliteHashCacher`` (persistent, production-grade) — both implementing -``CacherProtocol[FileHashKey, ContentHash]``. -""" - -import logging -import os -import sqlite3 -import threading -from pathlib import Path - -from orcapod.hashing.file_hashers import FileHashKey -from orcapod.types import ContentHash - -logger = logging.getLogger(__name__) - - -class InMemoryHashCacher: - """Dict-backed file hash cacher for testing and ephemeral in-process use. - - No persistence, no thread-safety guarantees beyond the GIL, no eviction. - Use ``SqliteHashCacher`` for production workloads. - """ - - def __init__(self) -> None: - self._cache: dict[FileHashKey, ContentHash] = {} - - def get(self, key: FileHashKey) -> ContentHash | None: - """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. - - Args: - key: File hash cache key. - - Returns: - Cached ``ContentHash``, or ``None`` if not found. - """ - return self._cache.get(key) - - def put(self, key: FileHashKey, value: ContentHash) -> None: - """Store ``value`` under ``key``. - - Args: - key: File hash cache key. - value: ``ContentHash`` to store. - """ - self._cache[key] = value - - def clear(self) -> None: - """Remove all entries from the cache.""" - self._cache.clear() - - -class SqliteHashCacher: - """SQLite-backed file hash cacher. - - Stores file hashes keyed on ``(path, mtime_ns, size)`` in a local - SQLite database. Uses WAL mode for single-writer/multi-reader - concurrency and thread-local connections for thread safety. - - The hash is stored as a BLOB in ``{method}:{raw_digest}`` format via - ``ContentHash.to_prefixed_digest()``. - - Args: - db_path: Path to the SQLite database file. Defaults to - ``~/.orcapod/file_hash_cache.db`` or the - ``ORCAPOD_HASH_CACHE_DB`` environment variable. - - Note: - Heavy multi-writer scenarios are a known SQLite limitation. A Turso - / libSQL migration is planned as a follow-up issue. - """ - - DEFAULT_DB_PATH = Path.home() / ".orcapod" / "file_hash_cache.db" - - def __init__(self, db_path: Path | None = None) -> None: - self.db_path = Path( - db_path - or os.environ.get("ORCAPOD_HASH_CACHE_DB") - or self.DEFAULT_DB_PATH - ) - self._local = threading.local() - self._ensure_schema() - - def _ensure_schema(self) -> None: - """Create the cache table and enable WAL mode. - - Uses a dedicated one-shot connection so schema setup happens once - on construction, independent of the thread-local connection pool. - """ - self.db_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.db_path) as conn: - conn.execute("PRAGMA journal_mode=WAL") - conn.execute( - """ - CREATE TABLE IF NOT EXISTS file_hash_cache ( - path TEXT NOT NULL, - mtime_ns INTEGER NOT NULL, - size INTEGER NOT NULL, - hash BLOB NOT NULL, - cached_at INTEGER NOT NULL DEFAULT (unixepoch()), - PRIMARY KEY (path, mtime_ns, size) - ) WITHOUT ROWID - """ - ) - conn.commit() - - def _connection(self) -> sqlite3.Connection: - """Return this thread's connection, opening it on first use.""" - conn = getattr(self._local, "conn", None) - if conn is None: - conn = sqlite3.connect(self.db_path) - conn.execute("PRAGMA journal_mode=WAL") - self._local.conn = conn - return conn - - def get(self, key: FileHashKey) -> ContentHash | None: - """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. - - Args: - key: File hash cache key. - - Returns: - Cached ``ContentHash``, or ``None`` if not found. - """ - conn = self._connection() - cursor = conn.execute( - "SELECT hash FROM file_hash_cache WHERE path=? AND mtime_ns=? AND size=?", - (str(key.path), key.mtime_ns, key.size), - ) - row = cursor.fetchone() - if row is None: - return None - blob: bytes = row[0] - method_bytes, digest = blob.split(b":", 1) - return ContentHash(method=method_bytes.decode("ascii"), digest=digest) - - def put(self, key: FileHashKey, value: ContentHash) -> None: - """Store ``value`` under ``key``. - - Uses ``INSERT OR REPLACE`` so writes are idempotent. - - Args: - key: File hash cache key. - value: ``ContentHash`` to store. - """ - conn = self._connection() - conn.execute( - """ - INSERT OR REPLACE INTO file_hash_cache (path, mtime_ns, size, hash) - VALUES (?, ?, ?, ?) - """, - (str(key.path), key.mtime_ns, key.size, value.to_prefixed_digest()), - ) - conn.commit() - - def clear(self) -> None: - """Delete all rows from the cache table.""" - conn = self._connection() - conn.execute("DELETE FROM file_hash_cache") - conn.commit() - - def close(self) -> None: - """Close this thread's database connection.""" - conn = getattr(self._local, "conn", None) - if conn is not None: - conn.close() - self._local.conn = None - - def __enter__(self) -> "SqliteHashCacher": - """Return self for use as a context manager.""" - return self - - def __exit__(self, *_: object) -> None: - """Close the thread-local connection on exit.""" - self.close() -``` - -- [ ] **Step 4: Run the cacher tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Run the file hasher tests (now `InMemoryHashCacher` exists)** - -```bash -uv run pytest tests/test_hashing/test_file_hashers.py tests/test_hashing/test_file_handler.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/hash_cachers.py \ - tests/test_hashing/test_hash_cachers.py -git commit -m "feat(hashing): add InMemoryHashCacher and SqliteHashCacher" -``` - ---- - -## Task 4: Update `hashing/__init__.py` exports - -**Files:** -- Modify: `src/orcapod/hashing/__init__.py` - -- [ ] **Step 1: Add new exports** - -Add the following import after the existing `file_hashers` import line: - -```python -from orcapod.hashing.hash_cachers import InMemoryHashCacher, SqliteHashCacher -``` - -Add to the `__all__` list (after `"CachedFileHasher"`): - -```python - "FileHashKey", - "InMemoryHashCacher", - "SqliteHashCacher", -``` - -Also add `"CacherProtocol"` to the imports from `hashing_protocols` and to `__all__`: - -```python -# In the protocols imports block, add: -from orcapod.protocols.hashing_protocols import ( - ArrowHasherProtocol, - CacherProtocol, # <-- add this - ContentIdentifiableProtocol, - ... -) -``` - -```python -# In __all__, add: - "CacherProtocol", -``` - -- [ ] **Step 2: Verify the public API imports cleanly** - -```bash -uv run python -c " -from orcapod.hashing import ( - CacherProtocol, FileHashKey, FileHasher, CachedFileHasher, - InMemoryHashCacher, SqliteHashCacher -) -print('all imports ok') -" -``` - -Expected: `all imports ok` - -- [ ] **Step 3: Run full hashing test suite** - -```bash -uv run pytest tests/test_hashing/ -q -``` - -Expected: all tests pass. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/hashing/__init__.py -git commit -m "feat(hashing): export CacherProtocol, FileHashKey, FileHasher, InMemoryHashCacher, SqliteHashCacher" -``` - ---- - -## Task 5: Add `enable_file_hash_caching()` to `contexts/__init__.py` - -**Files:** -- Modify: `src/orcapod/contexts/__init__.py` -- Modify: `tests/test_hashing/test_hash_cachers.py` - -- [ ] **Step 1: Write failing tests — append to `tests/test_hashing/test_hash_cachers.py`** - -```python -# --------------------------------------------------------------------------- -# enable_file_hash_caching() -# --------------------------------------------------------------------------- - - -@pytest.fixture() -def restore_default_file_handler(): - """Restore the default FileHandler after each test to prevent cross-test pollution.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.file_type import File - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - original_handler = registry.get_handler_for_type(File) - yield - registry.register(File, original_handler) - - -class TestEnableFileHashCaching: - def test_registers_cached_file_hasher(self, restore_default_file_handler, tmp_path): - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - from orcapod.hashing.file_hashers import CachedFileHasher - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - assert isinstance(handler.file_hasher, CachedFileHasher) - - def test_double_call_logs_warning_and_does_not_double_wrap( - self, restore_default_file_handler, tmp_path, caplog - ): - import logging - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - with caplog.at_level(logging.WARNING): - enable_file_hash_caching(db_path=tmp_path / "cache2.db") - - assert "already has a CachedFileHasher" in caplog.text - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - # Outer is CachedFileHasher - assert isinstance(handler.file_hasher, CachedFileHasher) - # Inner (the base hasher) must NOT be a CachedFileHasher - assert isinstance(handler.file_hasher.file_hasher, FileHasher) - - def test_preserves_base_hasher_algorithm( - self, restore_default_file_handler, tmp_path - ): - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - # The base FileHasher should still use sha256 (from v0.1.json) - assert handler.file_hasher.file_hasher.algorithm == "sha256" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching -x -q 2>&1 | head -20 -``` - -Expected: `ImportError: cannot import name 'enable_file_hash_caching'` - -- [ ] **Step 3: Implement `enable_file_hash_caching()` in `src/orcapod/contexts/__init__.py`** - -Add the following at the bottom of the imports section (before existing functions): - -```python -import logging as _logging - -_logger = _logging.getLogger(__name__) -``` - -Then add this function at the end of the file: - -```python -def enable_file_hash_caching(db_path: "Path | None" = None) -> None: - """Enable SQLite-backed file hash caching on the default Orcapod context. - - Wraps the existing ``FileHandler``'s hasher in a ``CachedFileHasher`` - backed by a ``SqliteHashCacher`` and re-registers it for ``orcapod.File`` - in the default context's semantic hasher registry. - - Call once at application startup before any file hashing occurs. - - If the handler already wraps a ``CachedFileHasher`` (i.e. this function - was already called), a warning is logged, all existing caching layers are - unwrapped to reach the original base hasher, and the new cacher is applied - around that base hasher. This keeps the system in a well-defined state - (exactly one caching layer) regardless of how many times this function - is called. - - For intentional multi-layer caching (e.g. in-memory L1 + SQLite L2), - construct a ``CachedFileHasher`` manually and register it directly via - the context's ``type_handler_registry`` instead. - - Args: - db_path: Path to the SQLite cache database. Defaults to - ``~/.orcapod/file_hash_cache.db`` or the - ``ORCAPOD_HASH_CACHE_DB`` environment variable. - """ - from pathlib import Path - - from orcapod.extension_types.file_type import File - from orcapod.hashing.file_hashers import CachedFileHasher - from orcapod.hashing.hash_cachers import SqliteHashCacher - from orcapod.hashing.semantic_hashing.builtin_handlers import FileHandler - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - - existing_handler = registry.get_handler_for_type(File) - if existing_handler is None: - raise RuntimeError( - "enable_file_hash_caching(): no FileHandler registered for " - "orcapod.File in the default context. This should not happen " - "with the standard v0.1 context." - ) - - base_hasher = existing_handler.file_hasher - - if isinstance(base_hasher, CachedFileHasher): - _logger.warning( - "enable_file_hash_caching() called but the default FileHandler " - "already has a CachedFileHasher. Unwrapping and replacing with " - "the new cacher. If layered caching is intentional, construct a " - "CachedFileHasher manually instead." - ) - while isinstance(base_hasher, CachedFileHasher): - base_hasher = base_hasher.file_hasher - - registry.register( - File, - FileHandler( - CachedFileHasher( - file_hasher=base_hasher, - cacher=SqliteHashCacher(db_path), - ) - ), - ) -``` - -- [ ] **Step 4: Run the new tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching -v -``` - -Expected: all 3 tests pass. - -- [ ] **Step 5: Run the full test suite** - -```bash -uv run pytest tests/ -q --tb=short 2>&1 | tail -20 -``` - -Expected: all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/contexts/__init__.py \ - tests/test_hashing/test_hash_cachers.py -git commit -m "feat(contexts): add enable_file_hash_caching() to wire SqliteHashCacher into default context" -``` - ---- - -## Task 6: Write the bench script - -**Files:** -- Create: `bench/bench_file_hasher_cache.py` - -- [ ] **Step 1: Create `bench/` directory and write bench script** - -```python -#!/usr/bin/env python -"""Benchmark: cached vs uncached file hashing. - -Creates a ≥100 MB temp file, then times three scenarios: - 1. Uncached hash (baseline) - 2. First cached hash (miss — hash computed + stored) - 3. Second cached hash (hit — lookup only, should be sub-millisecond) - -Run: uv run python bench/bench_file_hasher_cache.py -""" - -import tempfile -import time -from pathlib import Path - -from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher -from orcapod.hashing.hash_cachers import SqliteHashCacher - -FILE_SIZE_MB = 100 -ITERATIONS = 3 - - -def create_large_file(path: Path, size_mb: int) -> None: - chunk = b"\x00" * (1024 * 1024) - with open(path, "wb") as f: - for _ in range(size_mb): - f.write(chunk) - - -def time_call(fn, *args) -> tuple[object, float]: - t0 = time.perf_counter() - result = fn(*args) - elapsed = time.perf_counter() - t0 - return result, elapsed - - -def main() -> None: - with tempfile.TemporaryDirectory() as tmpdir: - tmp = Path(tmpdir) - large_file = tmp / "bench_input.bin" - db_path = tmp / "bench_cache.db" - - print(f"Creating {FILE_SIZE_MB} MB temp file...") - create_large_file(large_file, FILE_SIZE_MB) - - base_hasher = FileHasher(algorithm="sha256") - - # --- Uncached baseline --- - _, uncached_time = time_call(base_hasher.hash_file, large_file) - - # --- Cached: miss --- - cached_hasher = CachedFileHasher( - file_hasher=base_hasher, - cacher=SqliteHashCacher(db_path), - ) - _, miss_time = time_call(cached_hasher.hash_file, large_file) - - # --- Cached: hit (multiple times) --- - hit_times = [] - for _ in range(ITERATIONS): - _, t = time_call(cached_hasher.hash_file, large_file) - hit_times.append(t) - avg_hit = sum(hit_times) / len(hit_times) - - print() - print(f"{'Scenario':<30} {'Time (ms)':>12}") - print("-" * 44) - print(f"{'Uncached (baseline)':<30} {uncached_time * 1000:>12.2f}") - print(f"{'Cached miss (1st call)':<30} {miss_time * 1000:>12.2f}") - print(f"{'Cached hit (avg of 3)':<30} {avg_hit * 1000:>12.3f}") - print() - speedup = uncached_time / avg_hit if avg_hit > 0 else float("inf") - print(f"Cache hit speedup: {speedup:.0f}x") - if avg_hit < 0.001: - print("✓ Sub-millisecond cache hit achieved") - else: - print(f"! Cache hit is {avg_hit * 1000:.2f} ms — expected < 1 ms") - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 2: Run the bench script** - -```bash -uv run python bench/bench_file_hasher_cache.py -``` - -Expected output (timings will vary): -``` -Creating 100 MB temp file... - -Scenario Time (ms) --------------------------------------------- -Uncached (baseline) XXX.XX -Cached miss (1st call) XXX.XX -Cached hit (avg of 3) 0.XXX - -Cache hit speedup: XXXx -✓ Sub-millisecond cache hit achieved -``` - -- [ ] **Step 3: Run full test suite one final time** - -```bash -uv run pytest tests/ -q --tb=short 2>&1 | tail -10 -``` - -Expected: all tests pass, no failures. - -- [ ] **Step 4: Commit** - -```bash -git add bench/bench_file_hasher_cache.py -git commit -m "feat(bench): add file hasher cache benchmark script" -``` - ---- - -## Self-Review - -**Spec coverage check:** - -| Spec requirement | Covered by | -|---|---| -| `CacherProtocol[K, V]` protocol | Task 1 | -| `FileHashKey` frozen dataclass with `UPath` | Task 2 | -| `FileHasher` (rename of `BasicFileHasher`) | Task 2 | -| `CachedFileHasher` uses `CacherProtocol` + UPath normalization | Task 2 | -| `InMemoryHashCacher` | Task 3 | -| `SqliteHashCacher` with WAL, WITHOUT ROWID, BLOB, thread-local | Task 3 | -| `hashing/__init__.py` exports updated | Task 4 | -| `enable_file_hash_caching()` with warning on double-call | Task 5 | -| Tests: hit/miss/isolation/persistence/WAL/env-var | Task 3 | -| Tests: `enable_file_hash_caching()` double-call warning | Task 5 | -| Bench script on ≥100 MB file | Task 6 | -| All `BasicFileHasher` references updated | Task 2 | - -**All spec requirements covered. No gaps.** diff --git a/superpowers/plans/2026-07-02-si-sorting-logical-type.md b/superpowers/plans/2026-07-02-si-sorting-logical-type.md deleted file mode 100644 index 8ec385c58..000000000 --- a/superpowers/plans/2026-07-02-si-sorting-logical-type.md +++ /dev/null @@ -1,820 +0,0 @@ -# SpikeInterface BaseSorting LogicalType Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `LogicalSISorting` and `SISortingHandler` to orcapod so pods can natively accept and return SpikeInterface `BaseSorting` objects without manual save/reload scaffolding. - -**Architecture:** Mirror the existing `LogicalSIRecording` / `SIRecordingHandler` pattern from `spikeinterface_types.py` (ITL-459). `BaseSorting` and `BaseRecording` share the same `BaseExtractor` serialization API, so the implementation is structurally identical — different class names, extension name `"spikeinterface.sorting"`, and sorting-specific error messages. Auto-registration in `v0.1.json` with `_optional: true` activates the types whenever `spikeinterface` is installed. - -**Tech Stack:** Python 3.12, PyArrow, Polars, SpikeInterface >= 0.101, pytest, `uv run` for all commands. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/extension_types/spikeinterface_types.py` | Modify (lines 34–36, append after line 268) | Add `LogicalSISorting`, `SISortingHandler`; extend `register_spikeinterface_types()` | -| `src/orcapod/extension_types/__init__.py` | Modify (lines 35–38, 67–68) | Export `LogicalSISorting` from conditional import block | -| `src/orcapod/contexts/data/v0.1.json` | Modify (lines 49–52, 110, 147–148) | Add `_optional` entries for sorting LogicalType and handler; add changelog entry | -| `tests/test_extension_types/test_spikeinterface_types.py` | Modify (append) | Add sorting-specific tests | - ---- - -### Task 1: Add `LogicalSISorting` and `SISortingHandler` to `spikeinterface_types.py` - -**Files:** -- Modify: `src/orcapod/extension_types/spikeinterface_types.py:34-36` (import line) and append after line 268 - -- [ ] **Step 1: Write the failing import test** - -Add this test to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_logical_si_sorting_importable(): - si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISorting - import pyarrow as pa - - lt = LogicalSISorting() - assert lt.logical_type_name == "spikeinterface.sorting" - assert lt.python_type is si_core.BaseSorting - assert lt.get_arrow_extension_type().storage_type == pa.large_string() -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_sorting_importable -v -``` - -Expected: FAIL with `ImportError: cannot import name 'LogicalSISorting'` - -- [ ] **Step 3: Update the `spikeinterface.core` import line** - -In `src/orcapod/extension_types/spikeinterface_types.py`, replace line 35: - -```python -# Before (line 35): - from spikeinterface.core import BaseRecording - -# After: - from spikeinterface.core import BaseRecording, BaseSorting -``` - -- [ ] **Step 4: Append `LogicalSISorting` class after line 168 (after `LogicalSIRecording` class ends)** - -Append after the `LogicalSIRecording` class (before `class SIRecordingHandler`), i.e., insert between lines 168 and 171: - -```python - -class LogicalSISorting(BaseLogicalType): - """Logical type for ``spikeinterface.core.BaseSorting``. - - Stores ``BaseSorting`` instances as Arrow ``large_string`` columns - tagged with extension name ``"spikeinterface.sorting"``. The stored - value is SpikeInterface's own ``to_dict(recursive=True, - include_annotations=True, include_properties=False)`` output, encoded - via ``SIJsonEncoder``. Loading reconstructs the sorting via - ``spikeinterface.core.load(dict)``. - - Only sortings whose ``check_serializability("json")`` returns ``True`` - are accepted. File-backed sortings (zarr, numpy_folder, npz_folder, - sorter folder) qualify. In-memory ``NumpySorting`` objects do not and - raise ``ValueError`` with clear save instructions. - - Example: - >>> import tempfile, numpy as np - >>> import spikeinterface.core as si - >>> from orcapod.extension_types.spikeinterface_types import LogicalSISorting - >>> lt = LogicalSISorting() - >>> with tempfile.TemporaryDirectory() as tmp: - ... sorting = si.NumpySorting.from_unit_dict( - ... {0: np.array([0, 100, 200])}, sampling_frequency=30000 - ... ) - ... saved = sorting.save_to_folder(tmp + "/sorting") - ... storage = lt.python_to_storage(saved) - ... recovered = lt.storage_to_python(storage) - ... saved.get_unit_ids().tolist() == recovered.get_unit_ids().tolist() - True - """ - - _arrow_ext_class = make_arrow_extension_type("spikeinterface.sorting", pa.large_string()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("spikeinterface.sorting", pa.large_string()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "spikeinterface.sorting" - python_type: type = BaseSorting - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``BaseSorting``. - - Returns: - A ``pa.ExtensionType`` with extension name - ``"spikeinterface.sorting"`` and storage type ``pa.large_string()``. - """ - if LogicalSISorting._arrow_ext is None: - LogicalSISorting._arrow_ext = LogicalSISorting._arrow_ext_class() - return LogicalSISorting._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``BaseSorting``. - - Returns: - A ``pl.BaseExtension`` registered under ``"spikeinterface.sorting"``. - """ - if LogicalSISorting._polars_ext is None: - LogicalSISorting._polars_ext = LogicalSISorting._polars_ext_class() - return LogicalSISorting._polars_ext - - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None = None - ) -> str: - """Serialise a ``BaseSorting`` to its JSON storage representation. - - Args: - value: A ``BaseSorting`` instance whose - ``check_serializability("json")`` returns ``True``. - converter: Ignored. Present for protocol conformance. - - Returns: - A JSON string produced by ``sorting.to_dict(recursive=True, - include_annotations=True, include_properties=False)`` encoded - via ``SIJsonEncoder``. - - Raises: - ValueError: If the sorting is not JSON-serialisable (e.g. an - in-memory ``NumpySorting``). - """ - if not value.check_serializability("json"): - raise ValueError( - "This BaseSorting is not JSON-serializable and cannot be stored " - "by orcapod. This typically means it holds data in memory (e.g. " - "NumpySorting). Sortings built on top of file-backed data " - "(zarr, numpy_folder, npz_folder, etc.) are fine and do not need " - "to be materialized first. If your sorting is in-memory, call " - "sorting.save_to_zarr(path) or sorting.save_to_folder(path) " - "first, then pass the returned extractor to the pod." - ) - from spikeinterface.core.core_tools import SIJsonEncoder - return json.dumps( - value.to_dict( - include_annotations=True, - include_properties=False, - recursive=True, - ), - cls=SIJsonEncoder, - ) - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None = None - ) -> BaseSorting: - """Reconstruct a ``BaseSorting`` from its JSON storage string. - - Args: - storage_value: A JSON string as stored in Arrow. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``BaseSorting`` instance reconstructed via - ``spikeinterface.core.load``. - - Raises: - ValueError: If ``storage_value`` is not valid JSON. - FileNotFoundError: If the backing zarr/folder no longer exists - (raised by SpikeInterface, propagated as-is). - """ - from spikeinterface.core import load as si_load - try: - si_dict = json.loads(storage_value) - except (json.JSONDecodeError, TypeError) as exc: - raise ValueError( - f"LogicalSISorting: cannot deserialise storage value " - f"{storage_value!r}; expected a JSON string." - ) from exc - return si_load(si_dict) -``` - -- [ ] **Step 5: Run test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_sorting_importable -v -``` - -Expected: PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add LogicalSISorting class (ITL-468)" -``` - ---- - -### Task 2: Add `SISortingHandler` - -**Files:** -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` (append after `SIRecordingHandler`) - -- [ ] **Step 1: Write the failing handler tests** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def _make_numpy_sorting(): - """Create a small in-memory NumpySorting for use as a test source.""" - import spikeinterface.core as si_core - rng = np.random.default_rng(42) - n_samples = 1000 - spike_trains = { - 0: np.sort(rng.choice(n_samples, 20, replace=False)), - 1: np.sort(rng.choice(n_samples, 15, replace=False)), - } - return si_core.NumpySorting.from_unit_dict(spike_trains, sampling_frequency=30_000) - - -def test_si_sorting_handler_hash_stability(tmp_path): - """Same sorting produces identical ContentHash across two calls.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SISortingHandler - from orcapod.types import ContentHash - - saved = _make_numpy_sorting().save_to_folder(str(tmp_path / "sorting")) - handler = SISortingHandler() - - h1 = handler.handle(saved, hasher=None) - h2 = handler.handle(saved, hasher=None) - - assert isinstance(h1, ContentHash) - assert h1 == h2 - - -def test_si_sorting_handler_hash_changes_with_content(tmp_path): - """Different sortings produce different ContentHash values.""" - si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SISortingHandler - - rng = np.random.default_rng(0) - n_samples = 1000 - sorting_a = si_core.NumpySorting.from_unit_dict( - {0: np.sort(rng.choice(n_samples, 20, replace=False))}, - sampling_frequency=30_000, - ).save_to_folder(str(tmp_path / "sorting_a")) - sorting_b = si_core.NumpySorting.from_unit_dict( - {0: np.sort(rng.choice(n_samples, 20, replace=False))}, - sampling_frequency=30_000, - ).save_to_folder(str(tmp_path / "sorting_b")) - - handler = SISortingHandler() - assert handler.handle(sorting_a, hasher=None) != handler.handle(sorting_b, hasher=None) - - -def test_si_sorting_handler_in_memory_raises(): - """SISortingHandler raises ValueError for in-memory sortings.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SISortingHandler - - sorting = _make_numpy_sorting() - handler = SISortingHandler() - with pytest.raises(ValueError, match="in-memory"): - handler.handle(sorting, hasher=None) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_si_sorting_handler_hash_stability tests/test_extension_types/test_spikeinterface_types.py::test_si_sorting_handler_hash_changes_with_content tests/test_extension_types/test_spikeinterface_types.py::test_si_sorting_handler_in_memory_raises -v -``` - -Expected: FAIL with `ImportError: cannot import name 'SISortingHandler'` - -- [ ] **Step 3: Append `SISortingHandler` class to `spikeinterface_types.py`** - -Insert after `SIRecordingHandler` class (after line 221), before `register_spikeinterface_types`: - -```python - -class SISortingHandler: - """Semantic hash handler for ``spikeinterface.core.BaseSorting``. - - Computes a SHA-256 ``ContentHash`` of the JSON bytes produced by - ``sorting.to_dict(recursive=True, include_annotations=True, - include_properties=False)`` encoded via ``SIJsonEncoder``. This is - identical to the bytes that ``LogicalSISorting`` stores in Arrow, so - hash input and storage representation are always consistent. - - The ``hasher`` argument is accepted for protocol conformance but not used — - hashing is done directly via ``hashlib.sha256`` to avoid overhead. - """ - - def handle(self, obj: Any, hasher: SemanticHasherProtocol | None) -> ContentHash: - """Return a SHA-256 ``ContentHash`` of the sorting's JSON dump. - - Args: - obj: A ``BaseSorting`` instance. - hasher: Accepted for protocol conformance; not used. - - Returns: - A ``ContentHash`` with ``method="sha256"`` and digest equal to the - SHA-256 of the JSON bytes from ``to_dict(recursive=True, - include_annotations=True, include_properties=False)`` encoded - via ``SIJsonEncoder``. - - Raises: - TypeError: If ``obj`` is not a ``BaseSorting``. - ValueError: If the sorting is not JSON-serialisable (in-memory). - """ - if not isinstance(obj, BaseSorting): - raise TypeError( - f"SISortingHandler: expected BaseSorting, got {type(obj)!r}" - ) - if not obj.check_serializability("json"): - raise ValueError( - "Cannot hash an in-memory BaseSorting " - "(check_serializability('json') is False). " - "Save it to disk first with save_to_zarr() or save_to_folder()." - ) - # TODO(ITL-468): phase 2 — also hash backing source directory contents - from spikeinterface.core.core_tools import SIJsonEncoder - json_bytes = json.dumps( - obj.to_dict(include_annotations=True, include_properties=False, recursive=True), - cls=SIJsonEncoder, - ).encode() - logger.debug("SISortingHandler: hashing %d JSON bytes", len(json_bytes)) - return ContentHash( - method="sha256", - digest=hashlib.sha256(json_bytes).digest(), - ) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_si_sorting_handler_hash_stability tests/test_extension_types/test_spikeinterface_types.py::test_si_sorting_handler_hash_changes_with_content tests/test_extension_types/test_spikeinterface_types.py::test_si_sorting_handler_in_memory_raises -v -``` - -Expected: all 3 PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add SISortingHandler (ITL-468)" -``` - ---- - -### Task 3: Add serialization tests for `LogicalSISorting` - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` (append) - -- [ ] **Step 1: Write the failing serialization tests** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_in_memory_sorting_raises(): - """NumpySorting (json=False) raises ValueError with clear instructions.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISorting - - sorting = _make_numpy_sorting() - lt = LogicalSISorting() - - with pytest.raises(ValueError, match="not JSON-serializable"): - lt.python_to_storage(sorting) - - with pytest.raises(ValueError, match="file-backed"): - lt.python_to_storage(sorting) - - -def test_folder_sorting_round_trip(tmp_path): - """numpy_folder-backed sorting round-trips through python_to_storage / storage_to_python.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISorting - - saved = _make_numpy_sorting().save_to_folder(str(tmp_path / "sorting")) - lt = LogicalSISorting() - - storage = lt.python_to_storage(saved) - assert isinstance(storage, str) - data = json.loads(storage) - assert "class" in data # SI dict always has a "class" key - - recovered = lt.storage_to_python(storage) - np.testing.assert_array_equal( - saved.get_unit_spike_train(unit_id=0, segment_index=0), - recovered.get_unit_spike_train(unit_id=0, segment_index=0), - ) - - -def test_zarr_sorting_round_trip(tmp_path): - """Zarr-backed sorting round-trips through python_to_storage / storage_to_python.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISorting - - saved = _make_numpy_sorting().save_to_zarr(str(tmp_path / "sorting.zarr")) - lt = LogicalSISorting() - - storage = lt.python_to_storage(saved) - assert isinstance(storage, str) - - recovered = lt.storage_to_python(storage) - np.testing.assert_array_equal( - saved.get_unit_spike_train(unit_id=0, segment_index=0), - recovered.get_unit_spike_train(unit_id=0, segment_index=0), - ) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_in_memory_sorting_raises tests/test_extension_types/test_spikeinterface_types.py::test_folder_sorting_round_trip tests/test_extension_types/test_spikeinterface_types.py::test_zarr_sorting_round_trip -v -``` - -Expected: `test_in_memory_sorting_raises` PASS (class exists from Task 1), `test_folder_sorting_round_trip` and `test_zarr_sorting_round_trip` PASS — if all pass now, skip to Step 4. - -- [ ] **Step 3: Run full test file to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v -``` - -Expected: all tests PASS - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_extension_types/test_spikeinterface_types.py -git commit -m "test(extension_types): add LogicalSISorting serialization round-trip tests (ITL-468)" -``` - ---- - -### Task 4: Extend `register_spikeinterface_types()` to include sorting - -**Files:** -- Modify: `src/orcapod/extension_types/spikeinterface_types.py:224-268` (`register_spikeinterface_types`) - -- [ ] **Step 1: Write the failing registration test** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_register_spikeinterface_types_includes_sorting(tmp_path): - """register_spikeinterface_types() wires LogicalSISorting and SISortingHandler - into the default context so they are found by type lookup.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - from orcapod.contexts import get_default_context - - register_spikeinterface_types() - ctx = get_default_context() - - saved = _make_numpy_sorting().save_to_folder(str(tmp_path / "sorting")) - - # LogicalType registered: type_converter can find it - arrow_type = ctx.type_converter.python_type_to_arrow_type(type(saved)) - assert arrow_type.extension_name == "spikeinterface.sorting" - - # Handler registered: semantic_hasher can find it - from orcapod.types import ContentHash - handler = ctx.semantic_hasher.type_handler_registry.get_handler(saved) - assert handler is not None - result = handler.handle(saved, hasher=None) - assert isinstance(result, ContentHash) - - # Full round-trip through python_to_storage / storage_to_python - storage = ctx.type_converter.python_to_storage(saved, type(saved)) - recovered = ctx.type_converter.storage_to_python(storage, type(saved)) - np.testing.assert_array_equal( - saved.get_unit_spike_train(unit_id=0, segment_index=0), - recovered.get_unit_spike_train(unit_id=0, segment_index=0), - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_register_spikeinterface_types_includes_sorting -v -``` - -Expected: FAIL — `arrow_type.extension_name` will be `None` or raise because `BaseSorting` is not yet registered. - -- [ ] **Step 3: Extend `register_spikeinterface_types()` in `spikeinterface_types.py`** - -Replace the entire `register_spikeinterface_types` function body (currently lines 224–268) with: - -```python -def register_spikeinterface_types(context: Any = None) -> None: - """Register SpikeInterface LogicalTypes into an orcapod ``DataContext``. - - Registers both ``LogicalSIRecording`` / ``SIRecordingHandler`` (ITL-459) - and ``LogicalSISorting`` / ``SISortingHandler`` (ITL-468). - - For the default context this is called automatically at startup (the - default ``v0.1.json`` context config lists all four with ``"_optional": true``, - so they are wired in whenever ``spikeinterface`` is installed). Call this - function explicitly only when working with a custom ``DataContext`` that was - not constructed from the default config. - - If ``context`` is ``None``, the default context (from - ``orcapod.contexts.get_default_context()``) is used. The function is - idempotent — calling it more than once on the same context is safe. - - Args: - context: A ``DataContext`` instance, or ``None`` to use the default. - - Example: - >>> from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - >>> register_spikeinterface_types() # no-op if default context already has SI types - """ - if context is None: - from orcapod.contexts import get_default_context - context = get_default_context() - - # --- Recording --- - lt_recording = LogicalSIRecording() - try: - context.type_converter.register_logical_type(lt_recording) - except ValueError as exc: - if "already bound to" not in str(exc): - raise - logger.debug( - "register_spikeinterface_types: LogicalSIRecording already registered, skipping" - ) - else: - logger.debug("register_spikeinterface_types: registered LogicalSIRecording") - - context.semantic_hasher.type_handler_registry.register(BaseRecording, SIRecordingHandler()) - - # --- Sorting --- - lt_sorting = LogicalSISorting() - try: - context.type_converter.register_logical_type(lt_sorting) - except ValueError as exc: - if "already bound to" not in str(exc): - raise - logger.debug( - "register_spikeinterface_types: LogicalSISorting already registered, skipping" - ) - else: - logger.debug("register_spikeinterface_types: registered LogicalSISorting") - - context.semantic_hasher.type_handler_registry.register(BaseSorting, SISortingHandler()) -``` - -- [ ] **Step 4: Run new test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_register_spikeinterface_types_includes_sorting -v -``` - -Expected: PASS - -- [ ] **Step 5: Run full test file to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v -``` - -Expected: all tests PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): extend register_spikeinterface_types() for BaseSorting (ITL-468)" -``` - ---- - -### Task 5: Update `extension_types/__init__.py` to export `LogicalSISorting` - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py:35-38, 67-68` - -- [ ] **Step 1: Update the conditional import block** - -In `src/orcapod/extension_types/__init__.py`, replace lines 34–38: - -```python -# Before: -# ITL-459 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import LogicalSIRecording, register_spikeinterface_types - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -```python -# After: -# ITL-459, ITL-468 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import ( - LogicalSIRecording, - LogicalSISorting, - register_spikeinterface_types, - ) - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -- [ ] **Step 2: Update `__all__` to include `LogicalSISorting`** - -In `src/orcapod/extension_types/__init__.py`, replace lines 67–68: - -```python -# Before: - # ITL-459 (conditional — only present when spikeinterface is installed) - *( ["LogicalSIRecording", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), -``` - -```python -# After: - # ITL-459, ITL-468 (conditional — only present when spikeinterface is installed) - *( ["LogicalSIRecording", "LogicalSISorting", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), -``` - -- [ ] **Step 3: Verify the export is accessible** - -```bash -uv run python -c " -from orcapod.extension_types import LogicalSISorting, LogicalSIRecording, register_spikeinterface_types -print('LogicalSISorting:', LogicalSISorting) -print('OK') -" -``` - -Expected: prints `LogicalSISorting: ` and `OK` - -- [ ] **Step 4: Run full test suite to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/ -v -``` - -Expected: all tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "feat(extension_types): export LogicalSISorting from extension_types package (ITL-468)" -``` - ---- - -### Task 6: Update `v0.1.json` to auto-register sorting types - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Add `LogicalSISorting` to the logical types list** - -In `src/orcapod/contexts/data/v0.1.json`, after the `LogicalSIRecording` entry (lines 48–52): - -```json - { - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSIRecording", - "_config": {}, - "_optional": true - }, -``` - -Add immediately after: - -```json - { - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSISorting", - "_config": {}, - "_optional": true - }, -``` - -- [ ] **Step 2: Add `SISortingHandler` to the handler list** - -In `src/orcapod/contexts/data/v0.1.json`, after the `SIRecordingHandler` entry (line 110): - -```json - [{"_type": "spikeinterface.core.BaseRecording", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SIRecordingHandler", "_config": {}, "_optional": true}], -``` - -Add immediately after: - -```json - [{"_type": "spikeinterface.core.BaseSorting", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SISortingHandler", "_config": {}, "_optional": true}], -``` - -- [ ] **Step 3: Add changelog entry** - -In `src/orcapod/contexts/data/v0.1.json`, append to the `"changelog"` array (after the last entry): - -```json - "Added spikeinterface.BaseSorting as a native value type via LogicalSISorting (large_string/JSON) and SISortingHandler (SHA-256 of JSON bytes); auto-registered when spikeinterface is installed via _optional entries in v0.1.json (ITL-468)" -``` - -- [ ] **Step 4: Verify JSON is valid** - -```bash -uv run python -c "import json; json.load(open('src/orcapod/contexts/data/v0.1.json')); print('JSON valid')" -``` - -Expected: `JSON valid` - -- [ ] **Step 5: Verify auto-registration works from a fresh context** - -```bash -uv run python -c " -# Reset default context to force reload from config -import orcapod.contexts as ctx_mod -ctx_mod._default_context = None # force fresh construction if attr exists - -from orcapod.contexts import get_default_context -ctx = get_default_context() - -import spikeinterface.core as si -import numpy as np -rng = np.random.default_rng(0) -import tempfile -with tempfile.TemporaryDirectory() as tmp: - sorting = si.NumpySorting.from_unit_dict( - {0: np.sort(rng.choice(1000, 10, replace=False))}, - sampling_frequency=30000 - ).save_to_folder(tmp + '/s') - arrow_type = ctx.type_converter.python_type_to_arrow_type(type(sorting)) - print('extension_name:', arrow_type.extension_name) - assert arrow_type.extension_name == 'spikeinterface.sorting', arrow_type.extension_name - print('Auto-registration OK') -" -``` - -Expected: prints `extension_name: spikeinterface.sorting` and `Auto-registration OK` - -- [ ] **Step 6: Run full test suite** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v -``` - -Expected: all tests PASS - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json -git commit -m "feat(contexts): auto-register LogicalSISorting and SISortingHandler in v0.1.json (ITL-468)" -``` - ---- - -### Task 7: Final integration — run the full test suite - -**Files:** None (verification only) - -- [ ] **Step 1: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -30 -``` - -Expected: all tests PASS, no regressions. - -- [ ] **Step 2: If any test fails, investigate and fix before proceeding** - -Common failure modes: -- `already bound to` error in registration tests → idempotency guard working, check test isolation -- `FileNotFoundError` in round-trip tests → `tmp_path` is cleaned up; ensure `saved` stays in scope for the full test -- Arrow extension type conflict → two `make_arrow_extension_type` calls with the same name create the same class; class-level cache (`_arrow_ext`) prevents double-registration - -- [ ] **Step 3: Push branch and open PR** - -```bash -git push -u origin eywalker/itl-468-support-spikeinterface-basesorting-objects -gh pr create \ - --base main \ - --title "feat(extension_types): support SpikeInterface BaseSorting objects (ITL-468)" \ - --body "$(cat <<'EOF' -## Summary - -- Adds `LogicalSISorting` logical type mapping `BaseSorting` ↔ Arrow `large_string` via SpikeInterface's `to_dict()` JSON dump -- Adds `SISortingHandler` for SHA-256 content hashing (phase 1) -- Extends `register_spikeinterface_types()` to also wire in sorting types -- Auto-registers both types in `v0.1.json` with `_optional: true` -- Round-trip tests for numpy_folder-backed and zarr-backed sortings; clear error for in-memory `NumpySorting` - -Closes ITL-468 - -## Test plan -- [ ] `uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v` — all pass -- [ ] `uv run pytest tests/ -v` — no regressions - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` diff --git a/superpowers/plans/2026-07-02-spikeinterface-motion.md b/superpowers/plans/2026-07-02-spikeinterface-motion.md deleted file mode 100644 index a35b84ebe..000000000 --- a/superpowers/plans/2026-07-02-spikeinterface-motion.md +++ /dev/null @@ -1,793 +0,0 @@ -# SpikeInterface Motion LogicalType Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `LogicalSIMotion` and `SIMotionHandler` so that `spikeinterface.core.motion.Motion` objects can be stored in and retrieved from orcapod databases portably, with no external folder dependency. - -**Architecture:** `Motion` is fully in-memory computed data (numpy arrays + two strings), so it is serialised as a self-contained `.npz` archive stored in a `pa.large_binary()` Arrow column. A private helper `_motion_to_npz_bytes()` is shared by the logical type and the hash handler so the stored bytes and hash input are always identical. - -**Tech Stack:** Python, NumPy (`.npz`), PyArrow (`large_binary`), Polars, SpikeInterface (`spikeinterface.core.motion.Motion`). - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/extension_types/spikeinterface_types.py` | Add `import io`, `import numpy as np`; add `Motion` to SI import; add `_motion_to_npz_bytes()`, `LogicalSIMotion`, `SIMotionHandler`; extend `register_spikeinterface_types()` and its docstring; update module docstring | -| `src/orcapod/extension_types/__init__.py` | Add `LogicalSIMotion` to the optional SI import block and `__all__` | -| `src/orcapod/contexts/data/v0.1.json` | Add `LogicalSIMotion` logical type entry; add `SIMotionHandler` handler entry; add changelog entry | -| `tests/test_extension_types/test_spikeinterface_types.py` | Add `_make_motion()` helper and 7 Motion tests | - ---- - -## Task 1: `LogicalSIMotion` + round-trip tests - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` - -- [ ] **Step 1.1: Add the `_make_motion` test helper and three failing tests** - -Append to `tests/test_extension_types/test_spikeinterface_types.py` (after the last existing test): - -```python -# ── Motion helpers & fixtures ────────────────────────────────────────────── - - -def _make_motion(seed: int = 42) -> "spikeinterface.core.motion.Motion": - """Create a small deterministic single-segment Motion for use in tests.""" - from spikeinterface.core.motion import Motion - - rng = np.random.default_rng(seed) - n_temporal, n_spatial = 100, 5 - displacement = rng.standard_normal((n_temporal, n_spatial)) - temporal_bins = np.linspace(0, 10, n_temporal) - spatial_bins = np.linspace(0, 3000, n_spatial) - return Motion(displacement, temporal_bins, spatial_bins, direction="y") - - -# ── LogicalSIMotion tests ────────────────────────────────────────────────── - - -def test_logical_si_motion_importable(): - """LogicalSIMotion has the expected extension name, python_type, and storage type.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSIMotion - from spikeinterface.core.motion import Motion - import pyarrow as pa - - lt = LogicalSIMotion() - assert lt.logical_type_name == "spikeinterface.motion" - assert lt.python_type is Motion - assert lt.get_arrow_extension_type().storage_type == pa.large_binary() - - -def test_si_motion_round_trip(): - """Single-segment Motion round-trips through python_to_storage / storage_to_python.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSIMotion - - motion = _make_motion() - lt = LogicalSIMotion() - - storage = lt.python_to_storage(motion) - assert isinstance(storage, bytes) - - recovered = lt.storage_to_python(storage) - assert recovered == motion - - -def test_si_motion_multi_segment_round_trip(): - """Multi-segment Motion round-trips correctly, preserving segment count and direction.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSIMotion - from spikeinterface.core.motion import Motion - - rng = np.random.default_rng(99) - n_t, n_s = 80, 4 - motion = Motion( - displacement=[rng.standard_normal((n_t, n_s)), rng.standard_normal((n_t, n_s))], - temporal_bins_s=[np.linspace(0, 8, n_t), np.linspace(0, 8, n_t)], - spatial_bins_um=np.linspace(0, 2000, n_s), - direction="z", - interpolation_method="linear", - ) - lt = LogicalSIMotion() - recovered = lt.storage_to_python(lt.python_to_storage(motion)) - - assert recovered == motion - assert recovered.num_segments == 2 - assert recovered.direction == "z" - assert recovered.interpolation_method == "linear" -``` - -- [ ] **Step 1.2: Run the three tests — expect failure** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "motion" -v 2>&1 | tail -20 -``` - -Expected: all three tests fail with `ImportError: cannot import name 'LogicalSIMotion'`. - -- [ ] **Step 1.3: Add `import io` and `import numpy as np` to `spikeinterface_types.py`** - -In `src/orcapod/extension_types/spikeinterface_types.py`, the current imports block is: - -```python -from __future__ import annotations - -import hashlib -import json -import logging -from typing import TYPE_CHECKING, Any -``` - -Replace it with: - -```python -from __future__ import annotations - -import hashlib -import io -import json -import logging -from typing import TYPE_CHECKING, Any - -import numpy as np -``` - -- [ ] **Step 1.4: Add `Motion` to the SpikeInterface import block** - -The current SI import block (lines ~38–44) reads: - -```python -try: - from spikeinterface.core import BaseRecording, BaseSorting -except ImportError as _exc: - raise ImportError( - "spikeinterface is not installed. " - "Install it with: pip install orcapod[spikeinterface]" - ) from _exc -``` - -Replace with: - -```python -try: - from spikeinterface.core import BaseRecording, BaseSorting - from spikeinterface.core.motion import Motion -except ImportError as _exc: - raise ImportError( - "spikeinterface is not installed. " - "Install it with: pip install orcapod[spikeinterface]" - ) from _exc -``` - -- [ ] **Step 1.5: Add the `_motion_to_npz_bytes` private helper** - -Insert this function immediately after the `logger = logging.getLogger(__name__)` line and before `class LogicalSIRecording`: - -```python -def _motion_to_npz_bytes(motion: Motion) -> bytes: - """Serialise a ``Motion`` to ``.npz`` bytes. - - Shared by ``LogicalSIMotion.python_to_storage`` and ``SIMotionHandler.handle`` - to guarantee the stored bytes and hash input are always identical. - - The ``.npz`` archive keys are: ``spatial_bins_um``, ``direction`` (length-1 - string array), ``interpolation_method`` (length-1 string array), - ``num_segments`` (length-1 int array), ``displacement_{i}`` and - ``temporal_bins_s_{i}`` for each segment index ``i``. - - Args: - motion: A ``Motion`` instance. - - Returns: - Raw bytes of a NumPy ``.npz`` archive. - """ - buf = io.BytesIO() - kwargs: dict[str, np.ndarray] = { - "spatial_bins_um": motion.spatial_bins_um, - "direction": np.array([motion.direction]), - "interpolation_method": np.array([motion.interpolation_method]), - "num_segments": np.array([motion.num_segments]), - } - for i in range(motion.num_segments): - kwargs[f"displacement_{i}"] = motion.displacement[i] - kwargs[f"temporal_bins_s_{i}"] = motion.temporal_bins_s[i] - np.savez(buf, **kwargs) - return buf.getvalue() -``` - -- [ ] **Step 1.6: Add the `LogicalSIMotion` class** - -Insert this class after the `LogicalSISorting` class and before `class SIRecordingHandler`. The full class: - -```python -class LogicalSIMotion(BaseLogicalType): - """Logical type for ``spikeinterface.core.motion.Motion``. - - Stores ``Motion`` instances as Arrow ``large_binary`` columns tagged with - extension name ``"spikeinterface.motion"``. The stored value is a NumPy - ``.npz`` archive produced by ``_motion_to_npz_bytes()``, containing all - displacement arrays, temporal/spatial bin arrays, and scalar metadata - (``direction``, ``interpolation_method``, ``num_segments``). Loading - reconstructs the ``Motion`` directly from those arrays. - - Because the entire content is embedded in the ``.npz`` bytes, no external - folder is required — stored ``Motion`` objects are portable wherever the - database is accessible. - - Example: - >>> import numpy as np - >>> from spikeinterface.core.motion import Motion - >>> from orcapod.extension_types.spikeinterface_types import LogicalSIMotion - >>> lt = LogicalSIMotion() - >>> motion = Motion(np.zeros((10, 3)), np.linspace(0, 1, 10), np.array([0.0, 1.0, 2.0])) - >>> recovered = lt.storage_to_python(lt.python_to_storage(motion)) - >>> recovered == motion - True - """ - - _arrow_ext_class = make_arrow_extension_type("spikeinterface.motion", pa.large_binary()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("spikeinterface.motion", pa.large_binary()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "spikeinterface.motion" - python_type: type = Motion - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``Motion``. - - Returns: - A ``pa.ExtensionType`` with extension name ``"spikeinterface.motion"`` - and storage type ``pa.large_binary()``. - """ - if LogicalSIMotion._arrow_ext is None: - LogicalSIMotion._arrow_ext = LogicalSIMotion._arrow_ext_class() - return LogicalSIMotion._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``Motion``. - - Returns: - A ``pl.BaseExtension`` registered under ``"spikeinterface.motion"``. - """ - if LogicalSIMotion._polars_ext is None: - LogicalSIMotion._polars_ext = LogicalSIMotion._polars_ext_class() - return LogicalSIMotion._polars_ext - - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None = None - ) -> bytes: - """Serialise a ``Motion`` to its ``.npz`` storage representation. - - Args: - value: A ``Motion`` instance. - converter: Ignored. Present for protocol conformance. - - Returns: - Raw bytes of a NumPy ``.npz`` archive. - """ - return _motion_to_npz_bytes(value) - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None = None - ) -> Motion: - """Reconstruct a ``Motion`` from its ``.npz`` storage bytes. - - Args: - storage_value: Raw ``.npz`` bytes as stored in Arrow ``large_binary``. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``Motion`` instance reconstructed from the stored arrays. - - Raises: - ValueError: If ``storage_value`` cannot be parsed as a valid - ``.npz`` archive or is missing expected keys. - """ - try: - d = np.load(io.BytesIO(bytes(storage_value)), allow_pickle=False) - except Exception as exc: - raise ValueError( - f"LogicalSIMotion: cannot deserialise storage value of type " - f"{type(storage_value)!r}; expected raw .npz bytes." - ) from exc - try: - n = int(d["num_segments"][0]) - return Motion( - displacement=[d[f"displacement_{i}"] for i in range(n)], - temporal_bins_s=[d[f"temporal_bins_s_{i}"] for i in range(n)], - spatial_bins_um=d["spatial_bins_um"], - direction=str(d["direction"][0]), - interpolation_method=str(d["interpolation_method"][0]), - ) - except KeyError as exc: - raise ValueError( - f"LogicalSIMotion: .npz archive is missing expected key {exc}. " - f"The archive may have been produced by a different version of orcapod." - ) from exc -``` - -- [ ] **Step 1.7: Run the three tests — expect all to pass** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "motion" -v 2>&1 | tail -20 -``` - -Expected: -``` -PASSED tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_motion_importable -PASSED tests/test_extension_types/test_spikeinterface_types.py::test_si_motion_round_trip -PASSED tests/test_extension_types/test_spikeinterface_types.py::test_si_motion_multi_segment_round_trip -3 passed -``` - -- [ ] **Step 1.8: Confirm no existing tests are broken** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v 2>&1 | tail -10 -``` - -Expected: all existing tests still pass. - -- [ ] **Step 1.9: Commit** - -```bash -git add \ - src/orcapod/extension_types/spikeinterface_types.py \ - tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add LogicalSIMotion with .npz storage (ITL-470)" -``` - ---- - -## Task 2: `SIMotionHandler` + hash tests - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` - -- [ ] **Step 2.1: Add three failing handler tests** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -# ── SIMotionHandler tests ────────────────────────────────────────────────── - - -def test_si_motion_handler_hash_stability(): - """Same Motion produces identical ContentHash across two calls.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SIMotionHandler - from orcapod.types import ContentHash - - motion = _make_motion() - handler = SIMotionHandler() - - h1 = handler.handle(motion, hasher=None) - h2 = handler.handle(motion, hasher=None) - - assert isinstance(h1, ContentHash) - assert h1.method == "sha256" - assert h1 == h2 - - -def test_si_motion_handler_hash_changes_with_content(): - """Different Motion objects (different seeds) produce different ContentHash values.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SIMotionHandler - - motion_a = _make_motion(seed=1) - motion_b = _make_motion(seed=2) - - handler = SIMotionHandler() - assert handler.handle(motion_a, hasher=None) != handler.handle(motion_b, hasher=None) - - -def test_si_motion_handler_type_error(): - """SIMotionHandler raises TypeError with class name in message for non-Motion input.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SIMotionHandler - - handler = SIMotionHandler() - with pytest.raises(TypeError, match="SIMotionHandler"): - handler.handle("not a motion", hasher=None) -``` - -- [ ] **Step 2.2: Run the three handler tests — expect failure** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "handler" -v 2>&1 | tail -20 -``` - -Expected: the three new tests fail with `ImportError: cannot import name 'SIMotionHandler'`. Existing handler tests for Recording and Sorting still pass. - -- [ ] **Step 2.3: Add the `SIMotionHandler` class** - -Insert this class after `LogicalSIMotion` and before `class SIRecordingHandler`: - -```python -class SIMotionHandler: - """Semantic hash handler for ``spikeinterface.core.motion.Motion``. - - Computes a SHA-256 ``ContentHash`` of the ``.npz`` bytes produced by - ``_motion_to_npz_bytes()``. This is identical to the bytes that - ``LogicalSIMotion`` stores in Arrow, so hash input and storage - representation are always consistent. - - The ``hasher`` argument is accepted for protocol conformance but not used — - hashing is done directly via ``hashlib.sha256`` to avoid overhead. - """ - - def handle(self, obj: Any, hasher: SemanticHasherProtocol | None) -> ContentHash: - """Return a SHA-256 ``ContentHash`` of the motion's ``.npz`` bytes. - - Args: - obj: A ``Motion`` instance. - hasher: Accepted for protocol conformance; not used. - - Returns: - A ``ContentHash`` with ``method="sha256"`` and digest equal to the - SHA-256 of the ``.npz`` bytes from ``_motion_to_npz_bytes()``. - - Raises: - TypeError: If ``obj`` is not a ``Motion``. - """ - if not isinstance(obj, Motion): - raise TypeError( - f"SIMotionHandler: expected Motion, got {type(obj)!r}" - ) - npz_bytes = _motion_to_npz_bytes(obj) - logger.debug("SIMotionHandler: hashing %d .npz bytes", len(npz_bytes)) - return ContentHash( - method="sha256", - digest=hashlib.sha256(npz_bytes).digest(), - ) -``` - -- [ ] **Step 2.4: Run all Motion tests — expect all to pass** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "motion" -v 2>&1 | tail -15 -``` - -Expected: all 6 Motion tests pass. - -- [ ] **Step 2.5: Run the full SI test file — confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v 2>&1 | tail -10 -``` - -Expected: all tests pass. - -- [ ] **Step 2.6: Commit** - -```bash -git add \ - src/orcapod/extension_types/spikeinterface_types.py \ - tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add SIMotionHandler for Motion content hashing (ITL-470)" -``` - ---- - -## Task 3: Registration wiring + integration test - -**Files:** -- Modify: `tests/test_extension_types/test_spikeinterface_types.py` -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 3.1: Add the integration test — expect failure** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_register_spikeinterface_types_includes_motion(): - """register_spikeinterface_types() wires LogicalSIMotion and SIMotionHandler - into the default context so they are found by type lookup.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - from orcapod.contexts import get_default_context - from spikeinterface.core.motion import Motion - from orcapod.types import ContentHash - - register_spikeinterface_types() - ctx = get_default_context() - - motion = _make_motion() - - # LogicalType registered: type_converter can find it by python type - arrow_type = ctx.type_converter.python_type_to_arrow_type(Motion) - assert arrow_type.extension_name == "spikeinterface.motion" - - # Handler registered: semantic_hasher can find it by instance - handler = ctx.semantic_hasher.type_handler_registry.get_handler(motion) - assert handler is not None - result = handler.handle(motion, hasher=None) - assert isinstance(result, ContentHash) - - # Full round-trip through context type_converter - storage = ctx.type_converter.python_to_storage(motion, Motion) - recovered = ctx.type_converter.storage_to_python(storage, Motion) - assert recovered == motion -``` - -Run it to confirm it fails: - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "register_spikeinterface_types_includes_motion" -v 2>&1 | tail -15 -``` - -Expected: FAIL — `LogicalSIMotion` is not registered in context, type lookup raises. - -- [ ] **Step 3.2: Extend `register_spikeinterface_types()` in `spikeinterface_types.py`** - -The current function ends with the Sorting section (lines ~455–472). Append the Motion section immediately before the closing of the function body. The existing Sorting block ends with: - -```python - # Handler registration silently replaces an existing entry, so always safe. - context.semantic_hasher.type_handler_registry.register(BaseSorting, SISortingHandler()) -``` - -After that line, add: - -```python - # --- Motion --- - lt_motion = LogicalSIMotion() - try: - context.type_converter.register_logical_type(lt_motion) - except ValueError as exc: - # A different LogicalSIMotion instance is already registered (e.g. - # auto-registered from v0.1.json at context creation time). That is - # fine — both instances are equivalent. Any other ValueError propagates. - if "already bound to" not in str(exc): - raise - logger.debug( - "register_spikeinterface_types: LogicalSIMotion already registered, skipping" - ) - else: - logger.debug("register_spikeinterface_types: registered LogicalSIMotion") - - # Handler registration silently replaces an existing entry, so always safe. - context.semantic_hasher.type_handler_registry.register(Motion, SIMotionHandler()) -``` - -Also update the function's docstring. The current first paragraph reads: - -``` - Registers both ``LogicalSIRecording`` / ``SIRecordingHandler`` (ITL-459) - and ``LogicalSISorting`` / ``SISortingHandler`` (ITL-468). -``` - -Replace with: - -``` - Registers ``LogicalSIRecording`` / ``SIRecordingHandler`` (ITL-459), - ``LogicalSISorting`` / ``SISortingHandler`` (ITL-468), and - ``LogicalSIMotion`` / ``SIMotionHandler`` (ITL-470). -``` - -- [ ] **Step 3.3: Run the integration test — it should pass now (register path)** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "register_spikeinterface_types_includes_motion" -v 2>&1 | tail -10 -``` - -Expected: PASS. - -- [ ] **Step 3.4: Update `extension_types/__init__.py` to export `LogicalSIMotion`** - -The current optional SI import block (lines ~34–42) reads: - -```python -# ITL-459, ITL-468 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import ( - LogicalSIRecording, - LogicalSISorting, - register_spikeinterface_types, - ) - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -Replace with: - -```python -# ITL-459, ITL-468, ITL-470 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import ( - LogicalSIRecording, - LogicalSISorting, - LogicalSIMotion, - register_spikeinterface_types, - ) - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -The `__all__` list already has the conditional expression. The current entry reads: - -```python - # ITL-459, ITL-468 (conditional — only present when spikeinterface is installed) - *( ["LogicalSIRecording", "LogicalSISorting", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), -``` - -Replace with: - -```python - # ITL-459, ITL-468, ITL-470 (conditional — only present when spikeinterface is installed) - *( ["LogicalSIRecording", "LogicalSISorting", "LogicalSIMotion", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), -``` - -- [ ] **Step 3.5: Add entries to `contexts/data/v0.1.json`** - -**Logical type entry** — insert after the `LogicalSISorting` block (after the closing `}` on line ~56, before the closing `]` of `logical_types`). The current last two SI entries are: - -```json - { - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSIRecording", - "_config": {}, - "_optional": true - }, - { - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSISorting", - "_config": {}, - "_optional": true - }, -``` - -Add the Motion entry immediately after `LogicalSISorting`: - -```json - { - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSIMotion", - "_config": {}, - "_optional": true - }, -``` - -**Handler entry** — the current last two SI handler entries (lines ~115–116) are: - -```json - [{"_type": "spikeinterface.core.BaseRecording", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SIRecordingHandler", "_config": {}, "_optional": true}], - [{"_type": "spikeinterface.core.BaseSorting", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SISortingHandler", "_config": {}, "_optional": true}], -``` - -Add the Motion handler entry immediately after `SISortingHandler`: - -```json - [{"_type": "spikeinterface.core.motion.Motion", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SIMotionHandler", "_config": {}, "_optional": true}], -``` - -**Changelog entry** — the `"changelog"` array in the `"metadata"` section (currently ends with the ITL-468 entry). Append: - -```json -"Added spikeinterface.core.motion.Motion as a native value type via LogicalSIMotion (large_binary/.npz) and SIMotionHandler (SHA-256 of .npz bytes); auto-registered when spikeinterface is installed via _optional entries in v0.1.json (ITL-470)" -``` - -- [ ] **Step 3.6: Update the module docstring in `spikeinterface_types.py`** - -The current module docstring starts with: - -``` -"""SpikeInterface LogicalTypes and handlers for orcapod (ITL-459, ITL-468). - -``LogicalSIRecording`` maps ``spikeinterface.core.BaseRecording`` ↔ Arrow -``large_string`` using SpikeInterface's own ``to_dict(recursive=True, -include_annotations=True, include_properties=False)`` JSON dump (encoded -via ``SIJsonEncoder``) as the storage envelope. ``SIRecordingHandler`` hashes -the same JSON bytes via SHA-256 for content identity. - -``LogicalSISorting`` maps ``spikeinterface.core.BaseSorting`` ↔ Arrow -``large_string`` using the same serialization approach. ``SISortingHandler`` -hashes the JSON bytes via SHA-256. - -This module requires the optional ``spikeinterface`` extras group: -``pip install orcapod[spikeinterface]`` - -Register SI types into the default orcapod context before using them in -pods: call ``register_spikeinterface_types()`` once at startup. -""" -``` - -Replace with: - -``` -"""SpikeInterface LogicalTypes and handlers for orcapod (ITL-459, ITL-468, ITL-470). - -``LogicalSIRecording`` maps ``spikeinterface.core.BaseRecording`` ↔ Arrow -``large_string`` using SpikeInterface's own ``to_dict(recursive=True, -include_annotations=True, include_properties=False)`` JSON dump (encoded -via ``SIJsonEncoder``) as the storage envelope. ``SIRecordingHandler`` hashes -the same JSON bytes via SHA-256 for content identity. - -``LogicalSISorting`` maps ``spikeinterface.core.BaseSorting`` ↔ Arrow -``large_string`` using the same serialization approach. ``SISortingHandler`` -hashes the JSON bytes via SHA-256. - -``LogicalSIMotion`` maps ``spikeinterface.core.motion.Motion`` ↔ Arrow -``large_binary`` using a self-contained NumPy ``.npz`` archive as the storage -envelope. All displacement arrays, bin arrays, and scalar metadata are embedded -in the archive — no external folder is required. ``SIMotionHandler`` hashes -the same ``.npz`` bytes via SHA-256. - -This module requires the optional ``spikeinterface`` extras group: -``pip install orcapod[spikeinterface]`` - -Register SI types into the default orcapod context before using them in -pods: call ``register_spikeinterface_types()`` once at startup. -""" -``` - -- [ ] **Step 3.7: Run the full SI test suite — confirm all 7 Motion tests pass** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v 2>&1 | tail -25 -``` - -Expected: all tests pass, including the 7 new Motion tests: -``` -PASSED ...::test_logical_si_motion_importable -PASSED ...::test_si_motion_round_trip -PASSED ...::test_si_motion_multi_segment_round_trip -PASSED ...::test_si_motion_handler_hash_stability -PASSED ...::test_si_motion_handler_hash_changes_with_content -PASSED ...::test_si_motion_handler_type_error -PASSED ...::test_register_spikeinterface_types_includes_motion -``` - -- [ ] **Step 3.8: Run the broader extension_types tests to confirm no regressions** - -```bash -uv run pytest tests/test_extension_types/ -v 2>&1 | tail -10 -``` - -Expected: all tests pass. - -- [ ] **Step 3.9: Commit** - -```bash -git add \ - src/orcapod/extension_types/spikeinterface_types.py \ - src/orcapod/extension_types/__init__.py \ - src/orcapod/contexts/data/v0.1.json \ - tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): wire LogicalSIMotion registration and exports (ITL-470)" -``` - ---- - -## Self-review checklist - -- **Spec coverage:** - - ✓ `LogicalSIMotion` with `logical_type_name = "spikeinterface.motion"` — Task 1 - - ✓ `python_type = Motion`, no wrapper class — Task 1 - - ✓ `pa.large_binary()` storage type — Task 1 - - ✓ `.npz` archive with all named keys — Task 1 steps 1.5–1.6 - - ✓ `storage_to_python` via `np.load` with `allow_pickle=False` — Task 1 step 1.6 - - ✓ `ValueError` on malformed input — Task 1 step 1.6 - - ✓ `SIMotionHandler` SHA-256 of same `.npz` bytes — Task 2 - - ✓ `_motion_to_npz_bytes` shared helper — Task 1 step 1.5 - - ✓ `v0.1.json` logical type entry (`_optional: true`) — Task 3 step 3.5 - - ✓ `v0.1.json` handler entry (`_optional: true`) — Task 3 step 3.5 - - ✓ `__init__.py` export of `LogicalSIMotion` — Task 3 step 3.4 - - ✓ `register_spikeinterface_types()` extended — Task 3 step 3.2 - - ✓ All 7 tests from spec — Tasks 1, 2, 3 -- **No placeholders** — all steps contain complete code. -- **Type consistency** — `_motion_to_npz_bytes` defined in Task 1 step 1.5; called identically in `LogicalSIMotion` (step 1.6) and `SIMotionHandler` (step 2.3). diff --git a/superpowers/plans/2026-07-03-itl-489-warm-cache.md b/superpowers/plans/2026-07-03-itl-489-warm-cache.md deleted file mode 100644 index d785da4de..000000000 --- a/superpowers/plans/2026-07-03-itl-489-warm-cache.md +++ /dev/null @@ -1,899 +0,0 @@ -# Warm Cache Implementation Plan (ITL-489) - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `populate_hash_cache()` utility and `orcapod warm-cache` CLI subcommand that pre-populate the SQLite file-hash cache for large ephys recordings (default threshold: 500 MB). - -**Architecture:** A new `src/orcapod/hashing/cache_population.py` module provides `CachePopulationStats` and `populate_hash_cache()`, which walks a directory tree recursively (via an explicit DFS stack on `UPath.iterdir()`), skips files below the size threshold, and uses `SqliteHashCacher` + `FileHasher` from ITL-472 to hash and cache qualifying files. A new `src/orcapod/cli/` package provides the `orcapod` entry point (Typer app) with a `warm-cache` subcommand that wraps this function with human-readable CLI options and output. - -**Tech Stack:** Python 3.11, `upath.UPath`, `orcapod.hashing.file_hashers.{FileHasher,FileHashKey}`, `orcapod.hashing.hash_cachers.SqliteHashCacher`, `typer>=0.12`, `pytest`. - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/hashing/cache_population.py` | **NEW** — `CachePopulationStats`, `populate_hash_cache()` | -| `src/orcapod/hashing/__init__.py` | **Modify** — export `CachePopulationStats`, `populate_hash_cache` | -| `src/orcapod/cli/__init__.py` | **NEW** — Typer root app | -| `src/orcapod/cli/warm_cache.py` | **NEW** — `warm-cache` subcommand | -| `tests/test_hashing/test_cache_population.py` | **NEW** — unit tests for `populate_hash_cache` | -| `tests/test_cli/__init__.py` | **NEW** — empty, marks test package | -| `tests/test_cli/test_warm_cache.py` | **NEW** — CLI tests via `typer.testing.CliRunner` | -| `pyproject.toml` | **Modify** — add `typer>=0.12` dependency, add `[project.scripts]` | - ---- - -## Task 1: `CachePopulationStats` dataclass - -**Files:** -- Create: `src/orcapod/hashing/cache_population.py` -- Create: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 1.1: Create the test file with two failing tests** - -Create `tests/test_hashing/test_cache_population.py`: - -```python -"""Tests for populate_hash_cache() and CachePopulationStats.""" - -import pytest - - -class TestCachePopulationStats: - def test_instantiation(self): - from orcapod.hashing.cache_population import CachePopulationStats - - stats = CachePopulationStats( - hashed=1, - already_cached=2, - skipped_small=3, - errors=0, - total_bytes_hashed=100, - total_duration=1.0, - avg_hashing_speed=100.0, - ) - assert stats.hashed == 1 - assert stats.already_cached == 2 - assert stats.skipped_small == 3 - assert stats.errors == 0 - assert stats.total_bytes_hashed == 100 - assert stats.total_duration == 1.0 - assert stats.avg_hashing_speed == 100.0 - - def test_is_frozen(self): - from orcapod.hashing.cache_population import CachePopulationStats - - stats = CachePopulationStats( - hashed=0, - already_cached=0, - skipped_small=0, - errors=0, - total_bytes_hashed=0, - total_duration=0.0, - avg_hashing_speed=0.0, - ) - with pytest.raises(Exception): - stats.hashed = 1 # type: ignore[misc] -``` - -- [ ] **Step 1.2: Run tests — expect import failure** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -v -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'orcapod.hashing.cache_population'` - -- [ ] **Step 1.3: Create `cache_population.py` with just the dataclass** - -Create `src/orcapod/hashing/cache_population.py`: - -```python -"""File hash cache population utility. - -Provides ``CachePopulationStats`` and ``populate_hash_cache()`` for pre-populating -the SQLite file-hash cache with hashes of large files before a pipeline run. -""" - -from __future__ import annotations - -import logging -import time -from dataclasses import dataclass -from pathlib import Path - -from upath import UPath - -from orcapod.hashing.file_hashers import FileHashKey, FileHasher -from orcapod.hashing.hash_cachers import SqliteHashCacher -from orcapod.types import PathLike - -logger = logging.getLogger(__name__) - -_DEFAULT_MIN_SIZE_BYTES: int = 500 * 1024 * 1024 # 500 MB - - -@dataclass(frozen=True) -class CachePopulationStats: - """Statistics returned by ``populate_hash_cache()``. - - Attributes: - hashed: Files newly hashed and written to the cache. - already_cached: Files whose hash was already in the cache (rehash skipped). - skipped_small: Files skipped because their size was below ``min_size_bytes``. - errors: Files or directories that raised an exception during traversal or hashing - (logged as warnings). - total_bytes_hashed: Sum of bytes across all newly hashed files. - total_duration: Wall-clock duration of the full run in seconds. - avg_hashing_speed: Hashing throughput in bytes per second - (``total_bytes_hashed / total_duration``; ``0.0`` if ``total_duration`` is zero). - """ - - hashed: int - already_cached: int - skipped_small: int - errors: int - total_bytes_hashed: int - total_duration: float - avg_hashing_speed: float -``` - -- [ ] **Step 1.4: Run tests — expect pass** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestCachePopulationStats -v -``` - -Expected: 2 passed. - -- [ ] **Step 1.5: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): add CachePopulationStats dataclass (ITL-489)" -``` - ---- - -## Task 2: `populate_hash_cache()` — full implementation - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` -- Modify: `tests/test_hashing/test_cache_population.py` - -All tests use `min_size_bytes=10` so small test files (≥ 10 bytes) qualify without creating real 500 MB files. - -- [ ] **Step 2.1: Append all unit tests to the test file** - -Append to `tests/test_hashing/test_cache_population.py`: - -```python -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -_MIN = 10 # small threshold so test files qualify without being 500 MB - - -def _write(path, name: str, size: int): - """Write a file of exactly ``size`` bytes under ``path``.""" - f = path / name - f.write_bytes(b"x" * size) - return f - - -# --------------------------------------------------------------------------- -# Traversal & filtering -# --------------------------------------------------------------------------- - - -class TestTraversal: - def test_skips_files_below_threshold(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "small.bin", 5) # < 10 - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.skipped_small == 1 - assert stats.hashed == 0 - - def test_hashes_file_at_exact_threshold(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "exact.bin", 10) # == 10 - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.hashed == 1 - assert stats.skipped_small == 0 - - def test_recursive_traversal(self, tmp_path): - db = tmp_path / "cache.db" - sub = tmp_path / "a" / "b" - sub.mkdir(parents=True) - _write(sub, "deep.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.hashed == 1 - - def test_includes_hidden_files(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, ".hidden.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.hashed == 1 - - def test_skips_symlinks(self, tmp_path): - db = tmp_path / "cache.db" - real = tmp_path / "real.bin" - real.write_bytes(b"x" * 20) - link = tmp_path / "link.bin" - link.symlink_to(real) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - # Only the real file, not the symlink - assert stats.hashed == 1 - - -# --------------------------------------------------------------------------- -# Cache hit / miss -# --------------------------------------------------------------------------- - - -class TestCacheHitMiss: - def test_cache_hit_skips_rehash(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - first = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - second = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert first.hashed == 1 - assert second.already_cached == 1 - assert second.hashed == 0 - - def test_total_bytes_hashed(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 30) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.hashed == 2 - assert stats.total_bytes_hashed == 50 - - -# --------------------------------------------------------------------------- -# Stats: timing & speed -# --------------------------------------------------------------------------- - - -class TestStats: - def test_duration_positive(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 100) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.total_duration > 0 - - def test_speed_matches_bytes_over_duration(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 100) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.avg_hashing_speed == pytest.approx( - stats.total_bytes_hashed / stats.total_duration - ) - - def test_speed_zero_when_nothing_hashed(self, tmp_path): - db = tmp_path / "cache.db" - # Empty directory — nothing to hash - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.avg_hashing_speed == 0.0 - - def test_db_path_none_uses_env_var(self, tmp_path, tmp_path_factory, monkeypatch): - default_db = tmp_path_factory.mktemp("orcapod_db") / "file_hash_cache.db" - monkeypatch.setenv("ORCAPOD_HASH_CACHE_DB", str(default_db)) - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=None) - - assert stats.hashed == 1 - assert default_db.exists() - - -# --------------------------------------------------------------------------- -# Error handling -# --------------------------------------------------------------------------- - - -class TestErrors: - def test_file_error_increments_errors(self, tmp_path, monkeypatch): - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing import file_hashers - - def _raise(self, path): - raise OSError("simulated hashing error") - - monkeypatch.setattr(file_hashers.FileHasher, "hash_file", _raise) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert stats.errors == 1 - assert stats.hashed == 0 -``` - -- [ ] **Step 2.2: Run tests — expect failures** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -v --tb=no -q -``` - -Expected: multiple FAILs with `ImportError` or `TypeError` (function not implemented yet). - -- [ ] **Step 2.3: Implement `populate_hash_cache()` — append to `cache_population.py`** - -Append the following to `src/orcapod/hashing/cache_population.py` (after the `CachePopulationStats` dataclass): - -```python -def populate_hash_cache( - path: PathLike, - *, - min_size_bytes: int = _DEFAULT_MIN_SIZE_BYTES, - db_path: UPath | PathLike | None = None, - algorithm: str = "sha256", - buffer_size: int = 65536, -) -> CachePopulationStats: - """Recursively hash and cache all files >= ``min_size_bytes`` under ``path``. - - Pre-populates the SQLite file-hash cache so that subsequent pipeline runs - can skip expensive content hashing with a sub-millisecond DB lookup. - - All files under ``path`` are visited recursively, including hidden files. - Symlinks are skipped. Per-file and per-directory errors are logged as warnings - and counted in ``CachePopulationStats.errors``. - - Args: - path: Root directory to scan recursively. - min_size_bytes: Files strictly smaller than this threshold (in bytes) are - skipped and not cached. Defaults to 500 MB (``500 * 1024 * 1024``). - db_path: Path to the SQLite hash cache database. Only local paths are - supported — SQLite cannot operate on remote filesystems. Defaults to - the ``ORCAPOD_HASH_CACHE_DB`` environment variable, or - ``~/.orcapod/file_hash_cache.db`` if unset. - algorithm: Hash algorithm passed to ``FileHasher``. Defaults to ``"sha256"``. - buffer_size: Read buffer size in bytes passed to ``FileHasher``. - Defaults to 65536. - - Returns: - ``CachePopulationStats`` with counts for hashed, cached, skipped, and - errored files plus throughput metrics. - """ - root = UPath(path) - _db_path: Path | None = Path(db_path) if db_path is not None else None - cacher = SqliteHashCacher(_db_path) - hasher = FileHasher(algorithm=algorithm, buffer_size=buffer_size) - - hashed = 0 - already_cached = 0 - skipped_small = 0 - error_count = 0 - total_bytes_hashed = 0 - - start = time.monotonic() - - # Explicit DFS stack — avoids recursion limits on deep trees. - stack: list[UPath] = [root] - while stack: - current = stack.pop() - try: - entries = list(current.iterdir()) - except PermissionError: - logger.warning( - "Cannot access directory %s: permission denied", current - ) - error_count += 1 - continue - - for entry in entries: - # Never follow symlinks — they can cause cycles. - if entry.is_symlink(): - continue - if entry.is_dir(): - stack.append(entry) - continue - if not entry.is_file(): - continue - - try: - resolved = entry.resolve() - stat = resolved.stat() - size = stat.st_size - - if size < min_size_bytes: - skipped_small += 1 - continue - - key = FileHashKey(resolved, stat.st_mtime_ns, size) - if cacher.get(key) is not None: - already_cached += 1 - continue - - content_hash = hasher.hash_file(resolved) - cacher.put(key, content_hash) - hashed += 1 - total_bytes_hashed += size - - except Exception: - logger.warning( - "Failed to process file %s", entry, exc_info=True - ) - error_count += 1 - - duration = time.monotonic() - start - speed = total_bytes_hashed / duration if duration > 0 else 0.0 - - return CachePopulationStats( - hashed=hashed, - already_cached=already_cached, - skipped_small=skipped_small, - errors=error_count, - total_bytes_hashed=total_bytes_hashed, - total_duration=duration, - avg_hashing_speed=speed, - ) -``` - -- [ ] **Step 2.4: Run all tests — expect pass** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -v -``` - -Expected: all tests pass. - -- [ ] **Step 2.5: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): implement populate_hash_cache() with size threshold and stats (ITL-489)" -``` - ---- - -## Task 3: Export from `orcapod.hashing` - -**Files:** -- Modify: `src/orcapod/hashing/__init__.py` - -- [ ] **Step 3.1: Write a failing import test** - -Append to `tests/test_hashing/test_cache_population.py`: - -```python -class TestPublicExports: - def test_importable_from_orcapod_hashing(self): - from orcapod.hashing import CachePopulationStats, populate_hash_cache - - assert callable(populate_hash_cache) - assert CachePopulationStats.__dataclass_fields__ # is a dataclass -``` - -- [ ] **Step 3.2: Run test — expect fail** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestPublicExports -v -``` - -Expected: FAIL — `ImportError: cannot import name 'CachePopulationStats' from 'orcapod.hashing'` - -- [ ] **Step 3.3: Add exports to `src/orcapod/hashing/__init__.py`** - -After the existing line: - -```python -from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher, FileHashKey -``` - -add: - -```python -from orcapod.hashing.cache_population import CachePopulationStats, populate_hash_cache -``` - -In `__all__`, after the `"SqliteHashCacher"` entry, add: - -```python - "CachePopulationStats", - "populate_hash_cache", -``` - -- [ ] **Step 3.4: Run test — expect pass** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestPublicExports -v -``` - -Expected: PASS. - -- [ ] **Step 3.5: Run full test suite for the module** - -```bash -uv run pytest tests/test_hashing/ -v --tb=short -q -``` - -Expected: all existing + new tests pass. - -- [ ] **Step 3.6: Commit** - -```bash -git add src/orcapod/hashing/__init__.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): export CachePopulationStats and populate_hash_cache from orcapod.hashing (ITL-489)" -``` - ---- - -## Task 4: `typer` dependency + `[project.scripts]` entry point - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 4.1: Add `typer` to `[project.dependencies]` in `pyproject.toml`** - -In `pyproject.toml`, find the `dependencies = [` list under `[project]` and append `"typer>=0.12"`: - -```toml -dependencies = [ - ...existing entries..., - "typer>=0.12", -] -``` - -- [ ] **Step 4.2: Add `[project.scripts]` section to `pyproject.toml`** - -Add this new section immediately after the `[project.optional-dependencies]` block: - -```toml -[project.scripts] -orcapod = "orcapod.cli:app" -``` - -- [ ] **Step 4.3: Sync dependencies** - -```bash -uv sync -``` - -Expected: typer and its dependencies (click, etc.) installed with no errors. - -- [ ] **Step 4.4: Verify typer is importable** - -```bash -uv run python -c "import typer; print(typer.__version__)" -``` - -Expected: prints typer version (≥ 0.12). - -- [ ] **Step 4.5: Commit** - -```bash -git add pyproject.toml -git commit -m "chore(deps): add typer>=0.12 and register orcapod CLI entry point (ITL-489)" -``` - ---- - -## Task 5: `orcapod.cli` package and `warm-cache` subcommand - -**Files:** -- Create: `src/orcapod/cli/__init__.py` -- Create: `src/orcapod/cli/warm_cache.py` -- Create: `tests/test_cli/__init__.py` -- Create: `tests/test_cli/test_warm_cache.py` - -- [ ] **Step 5.1: Create the CLI test file with failing tests** - -Create `tests/test_cli/__init__.py` (empty): - -```python -``` - -Create `tests/test_cli/test_warm_cache.py`: - -```python -"""CLI tests for ``orcapod warm-cache``.""" - -import pytest -from typer.testing import CliRunner - - -@pytest.fixture -def runner(): - return CliRunner() - - -class TestWarmCacheCLI: - def test_help_exits_zero(self, runner): - from orcapod.cli import app - - result = runner.invoke(app, ["warm-cache", "--help"]) - assert result.exit_code == 0 - assert "PATH" in result.output - - def test_basic_run(self, runner, tmp_path): - db = tmp_path / "cache.db" - f = tmp_path / "f.bin" - f.write_bytes(b"x" * 20) - - from orcapod.cli import app - - result = runner.invoke( - app, - [ - "warm-cache", - str(tmp_path), - "--min-size", "0.00002", # ~20 bytes in MB, so 20-byte file qualifies - "--db-path", str(db), - ], - ) - assert result.exit_code == 0, result.output - assert "hashed" in result.output - - def test_already_cached_on_second_run(self, runner, tmp_path): - db = tmp_path / "cache.db" - f = tmp_path / "f.bin" - f.write_bytes(b"x" * 20) - - from orcapod.cli import app - - args = [ - "warm-cache", - str(tmp_path), - "--min-size", "0.00002", - "--db-path", str(db), - ] - runner.invoke(app, args) # first run — populates cache - result = runner.invoke(app, args) # second run — all cached - - assert result.exit_code == 0 - assert "already cached" in result.output - - def test_min_size_default_shown_in_help(self, runner): - from orcapod.cli import app - - result = runner.invoke(app, ["warm-cache", "--help"]) - assert "500" in result.output # default 500 MB should appear -``` - -- [ ] **Step 5.2: Run CLI tests — expect import failures** - -```bash -uv run pytest tests/test_cli/ -v --tb=short -q -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'orcapod.cli'` - -- [ ] **Step 5.3: Create `src/orcapod/cli/__init__.py`** - -```python -"""Orcapod command-line interface. - -Provides the ``orcapod`` entry point. Sub-commands are registered below. - -Usage:: - - orcapod warm-cache /data/recordings --min-size 500 -""" - -import typer - -from orcapod.cli.warm_cache import warm_cache - -app = typer.Typer( - name="orcapod", - help="Orcapod pipeline utilities.", - no_args_is_help=True, -) - -app.command("warm-cache")(warm_cache) -``` - -- [ ] **Step 5.4: Create `src/orcapod/cli/warm_cache.py`** - -```python -"""``orcapod warm-cache`` subcommand. - -Pre-populates the SQLite file-hash cache for large files under a target -directory so that subsequent pipeline runs skip expensive content hashing. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Optional - -import typer - -_DEFAULT_MIN_SIZE_MB: float = 500.0 - - -def warm_cache( - path: str = typer.Argument(..., help="Root directory to scan recursively."), - min_size: float = typer.Option( - _DEFAULT_MIN_SIZE_MB, - "--min-size", - help="Minimum file size in MB. Files smaller than this are skipped. Default: 500 MB.", - show_default=True, - ), - db_path: Optional[str] = typer.Option( - None, - "--db-path", - help=( - "Path to the SQLite hash-cache database. " - "Defaults to $ORCAPOD_HASH_CACHE_DB or ~/.orcapod/file_hash_cache.db." - ), - ), - algorithm: str = typer.Option( - "sha256", - "--algorithm", - help="Hash algorithm (sha256, xxh64, md5, …). Default: sha256.", - show_default=True, - ), - buffer_size: int = typer.Option( - 65536, - "--buffer-size", - help="Read buffer size in bytes. Default: 65536.", - show_default=True, - ), -) -> None: - """Pre-populate the file-hash cache for large files under PATH. - - Recursively scans PATH and hashes every file that is at least MIN_SIZE MB. - Files already present in the cache are skipped. On completion, prints a - summary with counts and throughput. - """ - from orcapod.hashing.cache_population import populate_hash_cache - - min_size_bytes = int(min_size * 1024 * 1024) - _db_path: Path | None = Path(db_path) if db_path is not None else None - - typer.echo(f"Scanning {path} ...") - - stats = populate_hash_cache( - path, - min_size_bytes=min_size_bytes, - db_path=_db_path, - algorithm=algorithm, - buffer_size=buffer_size, - ) - - gb = stats.total_bytes_hashed / (1024**3) - speed_gb = stats.avg_hashing_speed / (1024**3) - min_size_display = f"{min_size:.4g} MB" - - typer.echo( - f"Done in {stats.total_duration:.1f}s — " - f"{stats.hashed} hashed ({gb:.2f} GB), " - f"{stats.already_cached} already cached, " - f"{stats.skipped_small} skipped (< {min_size_display}), " - f"{stats.errors} errors." - ) - if stats.hashed > 0: - typer.echo(f"Average hashing speed: {speed_gb:.2f} GB/s") -``` - -- [ ] **Step 5.5: Run CLI tests — expect pass** - -```bash -uv run pytest tests/test_cli/ -v -``` - -Expected: all 4 tests pass. - -- [ ] **Step 5.6: Run full test suite** - -```bash -uv run pytest tests/test_hashing/ tests/test_cli/ -v --tb=short -q -``` - -Expected: all tests pass. - -- [ ] **Step 5.7: Commit** - -```bash -git add src/orcapod/cli/__init__.py src/orcapod/cli/warm_cache.py \ - tests/test_cli/__init__.py tests/test_cli/test_warm_cache.py -git commit -m "feat(cli): add orcapod warm-cache subcommand via typer (ITL-489)" -``` - ---- - -## Task 6: End-to-end smoke test - -Verify the installed script is reachable as `orcapod warm-cache` via `uv run`. - -- [ ] **Step 6.1: Confirm entry point is registered** - -```bash -uv run orcapod --help -``` - -Expected output contains: -``` -Usage: orcapod [OPTIONS] COMMAND [ARGS]... - Orcapod pipeline utilities. -Commands: - warm-cache Pre-populate the file-hash cache for large files under PATH. -``` - -- [ ] **Step 6.2: Run `warm-cache` on a temp directory** - -```bash -TMPDIR=$(mktemp -d) -dd if=/dev/urandom of="$TMPDIR/test.bin" bs=1M count=1 2>/dev/null -uv run orcapod warm-cache "$TMPDIR" --min-size 0.5 --db-path "$TMPDIR/cache.db" -``` - -Expected: output like: -``` -Scanning /tmp/tmpXXXXXX ... -Done in 0.0s — 1 hashed (0.00 GB), 0 already cached, 0 skipped (< 0.5 MB), 0 errors. -Average hashing speed: ... -``` - -- [ ] **Step 6.3: Run again — confirm cache hit** - -```bash -uv run orcapod warm-cache "$TMPDIR" --min-size 0.5 --db-path "$TMPDIR/cache.db" -``` - -Expected: `1 already cached, 0 hashed`. - -- [ ] **Step 6.4: Commit verification note** - -No code changes needed. If Step 6.1–6.3 all pass, the feature is complete. - -```bash -git log --oneline -5 -``` - -Confirm the four feature commits are present: -1. `feat(hashing): add CachePopulationStats dataclass (ITL-489)` -2. `feat(hashing): implement populate_hash_cache() with size threshold and stats (ITL-489)` -3. `feat(hashing): export CachePopulationStats and populate_hash_cache from orcapod.hashing (ITL-489)` -4. `chore(deps): add typer>=0.12 and register orcapod CLI entry point (ITL-489)` -5. `feat(cli): add orcapod warm-cache subcommand via typer (ITL-489)` diff --git a/superpowers/plans/2026-07-03-itl-493-pathlike-conformance.md b/superpowers/plans/2026-07-03-itl-493-pathlike-conformance.md deleted file mode 100644 index 2e62039b7..000000000 --- a/superpowers/plans/2026-07-03-itl-493-pathlike-conformance.md +++ /dev/null @@ -1,459 +0,0 @@ -# PathLike Conformance for op.File and op.Directory — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `op.File` and `op.Directory` pass `isinstance(x, os.PathLike)` and work with `os.fspath()`, `open()`, and `pathlib.Path()` for local-backed instances. - -**Architecture:** Add `__fspath__(self) -> str` to both `File` and `Directory`; the method delegates to `os.fspath(self.__wrapped__)`. The `os.PathLike` ABC's `__subclasshook__` picks up `__fspath__` automatically — no explicit base-class change is needed. Remote-backed instances raise `TypeError` on `os.fspath()`, mirroring `UPath`'s own behaviour. - -**Tech Stack:** Python `os.PathLike` ABC, `upath.extensions.ProxyUPath`, `pytest`. - ---- - -## File map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/extension_types/file_type.py` | Modify | Add `import os`; add `File.__fspath__` | -| `src/orcapod/extension_types/directory_type.py` | Modify | Add `import os`; add `Directory.__fspath__` | -| `tests/test_extension_types/test_file_type.py` | Modify | Add `TestFilePathLike` class | -| `tests/test_extension_types/test_directory_type.py` | Modify | Add `TestDirectoryPathLike` class | - ---- - -## Task 0: Checkout feature branch - -- [ ] **Step 1: Create and switch to the feature branch** - -```bash -git checkout -b eywalker/itl-493-make-opfile-opdirectory-subclass-ospathlike -``` - -Expected: `Switched to a new branch 'eywalker/itl-493-make-opfile-opdirectory-subclass-ospathlike'` - ---- - -## Task 1: `File.__fspath__` with tests - -**Files:** -- Modify: `tests/test_extension_types/test_file_type.py` -- Modify: `src/orcapod/extension_types/file_type.py` - -- [ ] **Step 1: Add imports to the test file** - -Open `tests/test_extension_types/test_file_type.py`. The current imports are: - -```python -from __future__ import annotations - -import json - -import pytest -import pyarrow as pa - -from orcapod.extension_types.file_type import File, LogicalFile -``` - -Replace with: - -```python -from __future__ import annotations - -import json -import os -import pathlib - -import pytest -import pyarrow as pa -from upath import UPath - -from orcapod.extension_types.file_type import File, LogicalFile -``` - -- [ ] **Step 2: Add the failing test class** - -Append the following class to the end of `tests/test_extension_types/test_file_type.py`: - -```python -class TestFilePathLike: - def test_isinstance_pathlike(self, tmp_path): - p = tmp_path / "file.txt" - p.write_text("data") - f = File(p) - assert isinstance(f, os.PathLike) - - def test_fspath_returns_path_string(self, tmp_path): - p = tmp_path / "file.txt" - p.write_text("data") - f = File(p) - assert os.fspath(f) == str(p) - - def test_open_accepts_file(self, tmp_path): - p = tmp_path / "file.txt" - p.write_text("hello") - f = File(p) - with open(f) as fh: - assert fh.read() == "hello" - - def test_pathlib_path_accepts_file(self, tmp_path): - p = tmp_path / "file.txt" - p.write_text("data") - f = File(p) - assert pathlib.Path(f) == p - - def test_remote_backed_fspath_raises(self): - remote = File._from_upath(UPath("s3://bucket/key.csv")) - with pytest.raises(TypeError): - os.fspath(remote) - - def test_plain_proxy_upath_subclass_not_pathlike(self): - from upath.extensions import ProxyUPath - - class _Stub(ProxyUPath): - pass - - assert not issubclass(_Stub, os.PathLike) -``` - -- [ ] **Step 3: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py::TestFilePathLike -v -``` - -Expected: all 6 tests FAIL. `test_isinstance_pathlike` should fail with `AssertionError` -(isinstance returns False). `test_open_accepts_file` and `test_pathlib_path_accepts_file` -should fail with `TypeError`. `test_plain_proxy_upath_subclass_not_pathlike` should PASS -(it is already true — `_Stub` has no `__fspath__`). - -- [ ] **Step 4: Add `import os` to `file_type.py`** - -Open `src/orcapod/extension_types/file_type.py`. The current imports start with: - -```python -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Any, Self -``` - -Add `import os` so it reads: - -```python -from __future__ import annotations - -import json -import os -from typing import TYPE_CHECKING, Any, Self -``` - -- [ ] **Step 5: Add `__fspath__` to `File`** - -In `src/orcapod/extension_types/file_type.py`, the `File` class ends at line 91 with -`_from_upath`. Insert `__fspath__` between `__init__` and `_from_upath`: - -```python - def __fspath__(self) -> str: - """Return the file system path representation of the underlying ``UPath``. - - Succeeds for local-backed paths (``PosixUPath``, ``FilePath``) and returns - the local path string. Raises ``TypeError`` for remote-backed paths (S3, GCS, - engm, …), consistent with how ``UPath`` itself behaves for those backends. - - Returns: - The local filesystem path as a string. - - Raises: - TypeError: If the underlying path is remote-backed and not ``os.PathLike``. - """ - return os.fspath(self.__wrapped__) -``` - -After the edit the relevant portion of `File` should look like: - -```python -class File(ProxyUPath): - ... - - def __init__(self, *args: Any, follow_symlinks: bool = True, **kwargs: Any) -> None: - ... - - def __fspath__(self) -> str: - """Return the file system path representation of the underlying ``UPath``. - - Succeeds for local-backed paths (``PosixUPath``, ``FilePath``) and returns - the local path string. Raises ``TypeError`` for remote-backed paths (S3, GCS, - engm, …), consistent with how ``UPath`` itself behaves for those backends. - - Returns: - The local filesystem path as a string. - - Raises: - TypeError: If the underlying path is remote-backed and not ``os.PathLike``. - """ - return os.fspath(self.__wrapped__) - - @classmethod - def _from_upath(cls, upath: UPath, /) -> Self: - ... -``` - -- [ ] **Step 6: Run the tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py::TestFilePathLike -v -``` - -Expected: all 6 tests PASS. - -- [ ] **Step 7: Run the full file test suite to check for regressions** - -```bash -uv run pytest tests/test_extension_types/test_file_type.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/extension_types/file_type.py tests/test_extension_types/test_file_type.py -git commit -m "feat(extension_types): add __fspath__ to File for os.PathLike conformance (ITL-493)" -``` - ---- - -## Task 2: `Directory.__fspath__` with tests - -**Files:** -- Modify: `tests/test_extension_types/test_directory_type.py` -- Modify: `src/orcapod/extension_types/directory_type.py` - -- [ ] **Step 1: Add imports to the test file** - -Open `tests/test_extension_types/test_directory_type.py`. Find the existing imports at the top -and add `import os`, `import pathlib`, and `from upath import UPath`: - -```python -from __future__ import annotations - -import json -import os -import pathlib - -import pytest -import pyarrow as pa -from upath import UPath - -from orcapod.extension_types.directory_type import Directory, LogicalDirectory -``` - -(Add only what is missing — do not duplicate existing imports.) - -- [ ] **Step 2: Add the failing test class** - -Append the following class to the end of `tests/test_extension_types/test_directory_type.py`: - -```python -class TestDirectoryPathLike: - def test_isinstance_pathlike(self, tmp_path): - d = Directory(tmp_path) - assert isinstance(d, os.PathLike) - - def test_fspath_returns_path_string(self, tmp_path): - d = Directory(tmp_path) - assert os.fspath(d) == str(tmp_path) - - def test_pathlib_path_accepts_directory(self, tmp_path): - d = Directory(tmp_path) - assert pathlib.Path(d) == tmp_path - - def test_remote_backed_fspath_raises(self): - remote = Directory._from_upath(UPath("s3://bucket/prefix/")) - with pytest.raises(TypeError): - os.fspath(remote) -``` - -- [ ] **Step 3: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_directory_type.py::TestDirectoryPathLike -v -``` - -Expected: `test_isinstance_pathlike`, `test_fspath_returns_path_string`, and -`test_pathlib_path_accepts_directory` FAIL. `test_remote_backed_fspath_raises` should -also FAIL (TypeError is not raised yet — `Directory` has no `__fspath__` so `os.fspath` -raises for a different reason or succeeds unexpectedly). - -- [ ] **Step 4: Add `import os` to `directory_type.py`** - -Open `src/orcapod/extension_types/directory_type.py`. The current imports start with: - -```python -from __future__ import annotations - -import importlib -import json -import warnings -from collections.abc import Callable, Iterable -from typing import TYPE_CHECKING, Any, Self -``` - -Add `import os` so it reads: - -```python -from __future__ import annotations - -import importlib -import json -import os -import warnings -from collections.abc import Callable, Iterable -from typing import TYPE_CHECKING, Any, Self -``` - -- [ ] **Step 5: Add `__fspath__` to `Directory`** - -In `src/orcapod/extension_types/directory_type.py`, the `Directory` class ends at line 101 -with `_from_upath`. Insert `__fspath__` between `__init__` and `_from_upath`: - -```python - def __fspath__(self) -> str: - """Return the file system path representation of the underlying ``UPath``. - - Succeeds for local-backed paths (``PosixUPath``, ``FilePath``) and returns - the local path string. Raises ``TypeError`` for remote-backed paths (S3, GCS, - engm, …), consistent with how ``UPath`` itself behaves for those backends. - - Returns: - The local filesystem path as a string. - - Raises: - TypeError: If the underlying path is remote-backed and not ``os.PathLike``. - """ - return os.fspath(self.__wrapped__) -``` - -After the edit the relevant portion of `Directory` should look like: - -```python -class Directory(ProxyUPath): - ... - - def __init__(self, *args, ignore=None, **kwargs): - ... - - def __fspath__(self) -> str: - """Return the file system path representation of the underlying ``UPath``. - - Succeeds for local-backed paths (``PosixUPath``, ``FilePath``) and returns - the local path string. Raises ``TypeError`` for remote-backed paths (S3, GCS, - engm, …), consistent with how ``UPath`` itself behaves for those backends. - - Returns: - The local filesystem path as a string. - - Raises: - TypeError: If the underlying path is remote-backed and not ``os.PathLike``. - """ - return os.fspath(self.__wrapped__) - - @classmethod - def _from_upath(cls, upath: UPath, /) -> Self: - ... -``` - -- [ ] **Step 6: Run the tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_directory_type.py::TestDirectoryPathLike -v -``` - -Expected: all 4 tests PASS. - -- [ ] **Step 7: Run the full directory test suite to check for regressions** - -```bash -uv run pytest tests/test_extension_types/test_directory_type.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/extension_types/directory_type.py tests/test_extension_types/test_directory_type.py -git commit -m "feat(extension_types): add __fspath__ to Directory for os.PathLike conformance (ITL-493)" -``` - ---- - -## Task 3: Final verification and PR - -- [ ] **Step 1: Run the full test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: all tests PASS with no regressions. - -- [ ] **Step 2: Smoke-check `isinstance` and `os.fspath` at the REPL** - -```bash -uv run python -c " -import os, tempfile, pathlib -from orcapod.extension_types.file_type import File -from orcapod.extension_types.directory_type import Directory - -with tempfile.TemporaryDirectory() as d: - p = pathlib.Path(d) / 'test.txt' - p.write_text('hi') - f = File(p) - print('File isinstance:', isinstance(f, os.PathLike)) - print('File os.fspath:', os.fspath(f)) - print('open(File):', open(f).read()) - print('pathlib.Path(File):', pathlib.Path(f)) - dr = Directory(d) - print('Directory isinstance:', isinstance(dr, os.PathLike)) - print('Directory os.fspath:', os.fspath(dr)) - print('pathlib.Path(Directory):', pathlib.Path(dr)) -" -``` - -Expected output: -``` -File isinstance: True -File os.fspath: /tmp//test.txt -open(File): hi -pathlib.Path(File): /tmp//test.txt -Directory isinstance: True -Directory os.fspath: /tmp/ -pathlib.Path(Directory): /tmp/ -``` - -- [ ] **Step 3: Create PR against `main`** - -```bash -git push -u origin eywalker/itl-493-make-opfile-opdirectory-subclass-ospathlike -gh pr create \ - --title "feat(extension_types): add os.PathLike conformance to op.File and op.Directory (ITL-493)" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Adds `__fspath__` to `File` and `Directory`, delegating to `os.fspath(self.__wrapped__)`. -- `isinstance(f, os.PathLike)` now returns `True` for both classes via the `os.PathLike` ABC's `__subclasshook__`. -- Local-backed instances work with `open()`, `pathlib.Path()`, and `shutil` directly. -- Remote-backed instances (S3, GCS, engm, …) raise `TypeError` on `os.fspath()`, mirroring `UPath`'s own behaviour. -- `UPathProxy` is unchanged. - -Closes ITL-493 - -## Test plan -- [ ] `uv run pytest tests/test_extension_types/test_file_type.py -v` — all pass -- [ ] `uv run pytest tests/test_extension_types/test_directory_type.py -v` — all pass -- [ ] `uv run pytest tests/ -v` — no regressions -EOF -)" -``` diff --git a/superpowers/plans/2026-07-03-itl-concurrent-warm-cache.md b/superpowers/plans/2026-07-03-itl-concurrent-warm-cache.md deleted file mode 100644 index 32c68c872..000000000 --- a/superpowers/plans/2026-07-03-itl-concurrent-warm-cache.md +++ /dev/null @@ -1,512 +0,0 @@ -# Concurrent Warm Cache Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `max_workers: int = 4` to `populate_hash_cache()` and `--workers INT` to `orcapod warm-cache` so file hashing runs concurrently across a `ThreadPoolExecutor`, saturating NAS/HDD bandwidth. - -**Architecture:** The DFS directory traversal stays single-threaded on the main thread; as each qualifying file is found it is submitted to a `ThreadPoolExecutor`. A new private `_hash_one()` function handles all per-file work (stat, size check, cache get, hash, cache put) and returns a `(category, bytes)` result tuple. The main thread collects results via `as_completed()` and accumulates stats counters. `SqliteHashCacher` is already thread-safe via `threading.local()` connections — no locking changes needed. - -**Tech Stack:** Python stdlib `concurrent.futures` (no new dependencies), `upath.UPath`, existing `FileHasher` and `SqliteHashCacher`. - ---- - -## Scene-setting - -You are working in the `orcapod-python` repository. The relevant files are: - -- `src/orcapod/hashing/cache_population.py` — contains `CachePopulationStats` and `populate_hash_cache()` -- `src/orcapod/cli/warm_cache.py` — contains the `warm_cache()` typer command -- `tests/test_hashing/test_cache_population.py` — existing tests for `populate_hash_cache()` -- `tests/test_cli/test_warm_cache.py` — existing CLI tests - -Always run tests with `uv run pytest -v`, never plain `pytest`. - -The existing `populate_hash_cache()` processes each file inline inside a single-threaded DFS loop. The goal is to extract that per-file logic into `_hash_one()` and dispatch it to a thread pool. - ---- - -## Task 1: Add concurrent hashing to `populate_hash_cache()` - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` -- Test: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 1: Write the three failing tests** - -Add a new `TestConcurrency` class at the bottom of `tests/test_hashing/test_cache_population.py` (before `TestPublicExports`): - -```python -# --------------------------------------------------------------------------- -# Concurrency -# --------------------------------------------------------------------------- - - -class TestConcurrency: - def test_concurrent_hashes_multiple_files(self, tmp_path): - """ThreadPoolExecutor path hashes all qualifying files without losing any.""" - db = tmp_path / "cache.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 20) - _write(tmp_path, "c.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db, max_workers=4 - ) - - assert stats.hashed == 3 - assert stats.errors == 0 - - def test_max_workers_1_matches_serial(self, tmp_path): - """max_workers=1 and max_workers=4 produce identical CachePopulationStats.""" - db1 = tmp_path / "cache1.db" - db2 = tmp_path / "cache2.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 30) - - from orcapod.hashing.cache_population import populate_hash_cache - - serial = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db1, max_workers=1 - ) - concurrent = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db2, max_workers=4 - ) - - assert serial.hashed == concurrent.hashed - assert serial.already_cached == concurrent.already_cached - assert serial.skipped_small == concurrent.skipped_small - assert serial.errors == concurrent.errors - assert serial.total_bytes_hashed == concurrent.total_bytes_hashed - - def test_workers_default_is_4(self): - """The default value of max_workers is 4.""" - import inspect - - from orcapod.hashing.cache_population import populate_hash_cache - - sig = inspect.signature(populate_hash_cache) - assert sig.parameters["max_workers"].default == 4 -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestConcurrency -v -``` - -Expected: 3 failures — `populate_hash_cache` has no `max_workers` parameter yet. - -``` -FAILED tests/test_hashing/test_cache_population.py::TestConcurrency::test_concurrent_hashes_multiple_files -FAILED tests/test_hashing/test_cache_population.py::TestConcurrency::test_max_workers_1_matches_serial -FAILED tests/test_hashing/test_cache_population.py::TestConcurrency::test_workers_default_is_4 -``` - -- [ ] **Step 3: Implement the new `cache_population.py`** - -Replace the entire content of `src/orcapod/hashing/cache_population.py` with: - -```python -"""File hash cache population utility. - -Provides ``CachePopulationStats`` and ``populate_hash_cache()`` for pre-populating -the SQLite file-hash cache with hashes of large files before a pipeline run. -""" - -from __future__ import annotations - -import logging -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass -from pathlib import Path - -from upath import UPath - -from orcapod.hashing.file_hashers import FileHashKey, FileHasher -from orcapod.hashing.hash_cachers import SqliteHashCacher -from orcapod.types import PathLike - -logger = logging.getLogger(__name__) - -_DEFAULT_MIN_SIZE_BYTES: int = 500 * 1024 * 1024 # 500 MB -_DEFAULT_MAX_WORKERS: int = 4 - - -@dataclass(frozen=True) -class CachePopulationStats: - """Statistics returned by ``populate_hash_cache()``. - - Attributes: - hashed: Files newly hashed and written to the cache. - already_cached: Files whose hash was already in the cache (rehash skipped). - skipped_small: Files skipped because their size was below ``min_size_bytes``. - errors: Files or directories that raised an exception during traversal or hashing - (logged as warnings). - total_bytes_hashed: Sum of bytes across all newly hashed files. - total_duration: Wall-clock duration of the full run in seconds. - avg_hashing_speed: Hashing throughput in bytes per second - (``total_bytes_hashed / total_duration``; ``0.0`` if ``total_duration`` is zero). - """ - - hashed: int - already_cached: int - skipped_small: int - errors: int - total_bytes_hashed: int - total_duration: float - avg_hashing_speed: float - - -def _hash_one( - entry: UPath, - cacher: SqliteHashCacher, - hasher: FileHasher, - min_size_bytes: int, -) -> tuple[str, int]: - """Hash and cache a single file, returning a (category, bytes) result pair. - - Called from worker threads inside ``populate_hash_cache()``. ``cacher`` and - ``hasher`` are shared across threads; both are safe because ``SqliteHashCacher`` - uses ``threading.local()`` connections and ``FileHasher`` is stateless. - - Args: - entry: File path to process. - cacher: Shared ``SqliteHashCacher`` instance. - hasher: Shared ``FileHasher`` instance. - min_size_bytes: Files strictly smaller than this threshold are skipped. - - Returns: - A tuple ``(category, bytes_hashed)`` where ``category`` is one of - ``"hashed"``, ``"cached"``, ``"skipped_small"``, or ``"error"``, and - ``bytes_hashed`` is the file size for newly hashed files, ``0`` otherwise. - """ - try: - resolved = entry.resolve() - stat = resolved.stat() - if stat.st_size < min_size_bytes: - return ("skipped_small", 0) - key = FileHashKey(resolved, stat.st_mtime_ns, stat.st_size) - if cacher.get(key) is not None: - return ("cached", 0) - content_hash = hasher.hash_file(resolved) - cacher.put(key, content_hash) - return ("hashed", stat.st_size) - except Exception: - logger.warning("Failed to process file %s", entry, exc_info=True) - return ("error", 0) - - -def populate_hash_cache( - path: PathLike, - *, - min_size_bytes: int = _DEFAULT_MIN_SIZE_BYTES, - db_path: UPath | PathLike | None = None, - algorithm: str = "sha256", - buffer_size: int = 65536, - max_workers: int = _DEFAULT_MAX_WORKERS, -) -> CachePopulationStats: - """Recursively hash and cache all files >= ``min_size_bytes`` under ``path``. - - Pre-populates the SQLite file-hash cache so that subsequent pipeline runs - can skip expensive content hashing with a sub-millisecond DB lookup. - - All files under ``path`` are visited recursively, including hidden files. - Symlinks are skipped. Per-file and per-directory errors are logged as warnings - and counted in ``CachePopulationStats.errors``. - - Directory traversal runs on the calling thread. Per-file work (stat, cache - lookup, hashing, cache write) is dispatched to a ``ThreadPoolExecutor`` with - ``max_workers`` threads. Pass ``max_workers=1`` to restore single-threaded - behaviour. - - Args: - path: Root directory to scan recursively. - min_size_bytes: Files strictly smaller than this threshold (in bytes) are - skipped and not cached. Defaults to 500 MB (``500 * 1024 * 1024``). - db_path: Path to the SQLite hash cache database. Only local paths are - supported — SQLite cannot operate on remote filesystems. Defaults to - the ``ORCAPOD_HASH_CACHE_DB`` environment variable, or - ``~/.orcapod/file_hash_cache.db`` if unset. - algorithm: Hash algorithm passed to ``FileHasher``. Defaults to ``"sha256"``. - buffer_size: Read buffer size in bytes passed to ``FileHasher``. - Defaults to 65536. - max_workers: Number of threads for concurrent per-file hashing. Defaults - to 4, which is well-suited for NAS and HDD storage where more than 4 - parallel reads cause seek contention. Pass ``1`` for serial behaviour. - - Returns: - ``CachePopulationStats`` with counts for hashed, cached, skipped, and - errored files plus throughput metrics. - """ - root = UPath(path) - _db_path: Path | None = Path(db_path) if db_path is not None else None - hasher = FileHasher(algorithm=algorithm, buffer_size=buffer_size) - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - with SqliteHashCacher(_db_path) as cacher: - # Collect the DB file and its SQLite journal/WAL siblings so they are - # never treated as data files even if the DB lives inside the scan root. - _db_resolved = cacher.db_path.resolve() - _excluded: frozenset[Path] = frozenset( - [ - _db_resolved, - _db_resolved.with_suffix(_db_resolved.suffix + "-wal"), - _db_resolved.with_suffix(_db_resolved.suffix + "-shm"), - ] - ) - - error_count = 0 - start = time.monotonic() - - # Explicit DFS stack — avoids recursion limits on deep trees. - # DB-file exclusion happens here (main thread) so workers never - # receive those paths. - futures = [] - stack: list[UPath] = [root] - while stack: - current = stack.pop() - try: - entries = list(current.iterdir()) - except OSError: - logger.warning( - "Cannot access directory %s", current, exc_info=True - ) - error_count += 1 - continue - - for entry in entries: - if entry.is_symlink(): - continue - if entry.is_dir(): - stack.append(entry) - continue - if not entry.is_file(): - continue - if entry.resolve() in _excluded: - continue - futures.append( - executor.submit(_hash_one, entry, cacher, hasher, min_size_bytes) - ) - - # Collect results as workers complete. - hashed = 0 - already_cached = 0 - skipped_small = 0 - total_bytes_hashed = 0 - - for future in as_completed(futures): - category, nbytes = future.result() - if category == "hashed": - hashed += 1 - total_bytes_hashed += nbytes - elif category == "cached": - already_cached += 1 - elif category == "skipped_small": - skipped_small += 1 - else: # "error" - error_count += 1 - - duration = time.monotonic() - start - speed = total_bytes_hashed / duration if duration > 0 else 0.0 - - return CachePopulationStats( - hashed=hashed, - already_cached=already_cached, - skipped_small=skipped_small, - errors=error_count, - total_bytes_hashed=total_bytes_hashed, - total_duration=duration, - avg_hashing_speed=speed, - ) -``` - -- [ ] **Step 4: Run the new concurrency tests** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestConcurrency -v -``` - -Expected: all 3 pass. - -``` -PASSED tests/test_hashing/test_cache_population.py::TestConcurrency::test_concurrent_hashes_multiple_files -PASSED tests/test_hashing/test_cache_population.py::TestConcurrency::test_max_workers_1_matches_serial -PASSED tests/test_hashing/test_cache_population.py::TestConcurrency::test_workers_default_is_4 -``` - -- [ ] **Step 5: Run the full existing test suite to confirm no regressions** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -v -``` - -Expected: all tests pass (16 pre-existing + 3 new = 19 total). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): add concurrent hashing via ThreadPoolExecutor to populate_hash_cache" -``` - ---- - -## Task 2: Add `--workers` option to `orcapod warm-cache` - -**Files:** -- Modify: `src/orcapod/cli/warm_cache.py` -- Test: `tests/test_cli/test_warm_cache.py` - -- [ ] **Step 1: Write the failing CLI test** - -Add this test to the `TestWarmCacheCLI` class in `tests/test_cli/test_warm_cache.py`: - -```python - def test_workers_option_accepted(self, runner, tmp_path): - db = tmp_path / "cache.db" - - from orcapod.cli import app - - result = runner.invoke( - app, - [ - "warm-cache", - str(tmp_path), - "--min-size", "0", - "--workers", "2", - "--db-path", str(db), - ], - ) - assert result.exit_code == 0, result.output -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -uv run pytest tests/test_cli/test_warm_cache.py::TestWarmCacheCLI::test_workers_option_accepted -v -``` - -Expected: FAIL — `--workers` is not a recognised option yet. - -``` -FAILED tests/test_cli/test_warm_cache.py::TestWarmCacheCLI::test_workers_option_accepted -``` - -- [ ] **Step 3: Add `--workers` to `warm_cache.py`** - -The current `warm_cache()` function signature ends at `buffer_size`. Add `max_workers` and pass it through. The full updated function: - -```python -def warm_cache( - path: str = typer.Argument(..., help="Root directory to scan recursively."), - min_size: float = typer.Option( - _DEFAULT_MIN_SIZE_MB, - "--min-size", - help="Minimum file size in MB. Files smaller than this are skipped. Default: 500 MB.", - show_default=True, - ), - db_path: str | None = typer.Option( - None, - "--db-path", - help=( - "Path to the SQLite hash-cache database. " - "Defaults to $ORCAPOD_HASH_CACHE_DB or ~/.orcapod/file_hash_cache.db." - ), - ), - algorithm: str = typer.Option( - "sha256", - "--algorithm", - help="Hash algorithm (sha256, xxh64, md5, …). Default: sha256.", - show_default=True, - ), - buffer_size: int = typer.Option( - 65536, - "--buffer-size", - help="Read buffer size in bytes. Default: 65536.", - show_default=True, - ), - max_workers: int = typer.Option( - 4, - "--workers", - help="Number of threads for concurrent hashing. Default: 4.", - show_default=True, - ), -) -> None: - """Pre-populate the file-hash cache for large files under PATH. - - Recursively scans PATH and hashes every file that is at least MIN_SIZE MB. - Files already present in the cache are skipped. On completion, prints a - summary with counts and throughput. - """ - from orcapod.hashing.cache_population import populate_hash_cache - - min_size_bytes = int(min_size * 1024 * 1024) - _db_path: Path | None = Path(db_path) if db_path is not None else None - - root = Path(path) - if not root.exists(): - typer.echo(f"Error: path does not exist: {path}", err=True) - raise typer.Exit(code=1) - if not root.is_dir(): - typer.echo(f"Error: path is not a directory: {path}", err=True) - raise typer.Exit(code=1) - - typer.echo(f"Scanning {path} ...") - - stats = populate_hash_cache( - path, - min_size_bytes=min_size_bytes, - db_path=_db_path, - algorithm=algorithm, - buffer_size=buffer_size, - max_workers=max_workers, - ) - - gb = stats.total_bytes_hashed / (1024**3) - speed_gb = stats.avg_hashing_speed / (1024**3) - min_size_display = f"{min_size:g} MB" - - typer.echo( - f"Done in {stats.total_duration:.1f}s — " - f"{stats.hashed} hashed ({gb:.2f} GB), " - f"{stats.already_cached} already cached, " - f"{stats.skipped_small} skipped (< {min_size_display}), " - f"{stats.errors} errors." - ) - if stats.hashed > 0: - typer.echo(f"Average hashing speed: {speed_gb:.2f} GB/s") -``` - -- [ ] **Step 4: Run the new CLI test** - -```bash -uv run pytest tests/test_cli/test_warm_cache.py::TestWarmCacheCLI::test_workers_option_accepted -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run the full CLI test suite** - -```bash -uv run pytest tests/test_cli/ -v -``` - -Expected: all 6 tests pass (5 pre-existing + 1 new). - -- [ ] **Step 6: Run the complete test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/cli/warm_cache.py tests/test_cli/test_warm_cache.py -git commit -m "feat(cli): add --workers option to orcapod warm-cache" -``` diff --git a/superpowers/plans/2026-07-03-sorting-analyzer-logical-type.md b/superpowers/plans/2026-07-03-sorting-analyzer-logical-type.md deleted file mode 100644 index b00879b20..000000000 --- a/superpowers/plans/2026-07-03-sorting-analyzer-logical-type.md +++ /dev/null @@ -1,714 +0,0 @@ -# LogicalSISortingAnalyzer Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `LogicalSISortingAnalyzer` and `SISortingAnalyzerHandler` to orcapod as a registered LogicalType for SpikeInterface `SortingAnalyzer` objects. - -**Architecture:** `SortingAnalyzer` is folder-backed only (no `to_dict()`), so storage is a JSON object `{"folder": "...", "format": "binary_folder"|"zarr"}` stored as `pa.large_string()`. `storage_to_python` calls `SortingAnalyzer.load(folder)` which auto-detects format from the folder suffix. Hash = SHA-256 of `str(analyzer.folder).encode()` (phase 1; content hashing deferred to ITL-476). - -**Tech Stack:** Python 3.12, SpikeInterface ≥ 0.101, PyArrow, Polars, pytest, uv. - ---- - -## Files - -| File | Change | -|---|---| -| `src/orcapod/extension_types/spikeinterface_types.py` | Add `SortingAnalyzer` import, `LogicalSISortingAnalyzer`, `SISortingAnalyzerHandler`, update `register_spikeinterface_types()` and module docstring | -| `src/orcapod/extension_types/__init__.py` | Add `LogicalSISortingAnalyzer` and `SISortingAnalyzerHandler` to conditional SI imports and `__all__` | -| `src/orcapod/contexts/data/v0.1.json` | Add `_optional` entries for `LogicalSISortingAnalyzer` and `SISortingAnalyzerHandler`; add changelog entry | -| `tests/test_extension_types/test_spikeinterface_types.py` | Add `_make_sorting_analyzer()` helper + all 8 test cases | - ---- - -## Task 1: Add `LogicalSISortingAnalyzer` class skeleton + importable test - -**Files:** -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` -- Test: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Write the failing test** - -Append to the end of `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -# ── SortingAnalyzer helpers ─────────────────────────────────────────────────── - -def _make_sorting_analyzer(tmp_path, fmt: str): - """Create a minimal saved SortingAnalyzer for use as a test artifact. - - Args: - tmp_path: Base directory. Created if it does not exist. - fmt: ``"binary_folder"`` or ``"zarr"``. - - Returns: - A ``SortingAnalyzer`` saved to ``tmp_path/analyzer`` (binary_folder) - or ``tmp_path/analyzer.zarr`` (zarr). - """ - import spikeinterface.core as si_core - from pathlib import Path - tmp_path = Path(tmp_path) - tmp_path.mkdir(parents=True, exist_ok=True) - rng = np.random.default_rng(42) - traces = rng.standard_normal((200, 4)).astype("float32") - recording = si_core.NumpyRecording([traces], sampling_frequency=30_000) - spike_trains = {0: np.sort(rng.choice(200, 10, replace=False))} - sorting = si_core.NumpySorting.from_unit_dict(spike_trains, sampling_frequency=30_000) - if fmt == "zarr": - folder = str(tmp_path / "analyzer.zarr") - else: - folder = str(tmp_path / "analyzer") - return si_core.SortingAnalyzer.create(sorting, recording, format=fmt, folder=folder) - - -# ── LogicalSISortingAnalyzer tests ──────────────────────────────────────────── - -def test_logical_si_sorting_analyzer_importable(): - """``LogicalSISortingAnalyzer`` exposes the expected extension name, python_type, - and Arrow storage type.""" - si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISortingAnalyzer - import pyarrow as pa - - lt = LogicalSISortingAnalyzer() - assert lt.logical_type_name == "spikeinterface.sorting_analyzer" - assert lt.python_type is si_core.SortingAnalyzer - assert lt.get_arrow_extension_type().storage_type == pa.large_string() -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_sorting_analyzer_importable -v -``` - -Expected: `FAILED` — `ImportError` or `AttributeError` (class does not exist yet). - -- [ ] **Step 3: Import `SortingAnalyzer` in `spikeinterface_types.py`** - -In `src/orcapod/extension_types/spikeinterface_types.py`, change the import block at lines 48–54: - -```python -try: - from spikeinterface.core import BaseRecording, BaseSorting, SortingAnalyzer - from spikeinterface.core.motion import Motion -except ImportError as _exc: - raise ImportError( - "spikeinterface is not installed. " - "Install it with: pip install orcapod[spikeinterface]" - ) from _exc -``` - -- [ ] **Step 4: Add `LogicalSISortingAnalyzer` class** - -Append the following after `class SISortingHandler` (after line 597) and before `def register_spikeinterface_types`: - -```python -class LogicalSISortingAnalyzer(BaseLogicalType): - """Logical type for ``spikeinterface.core.SortingAnalyzer``. - - Stores ``SortingAnalyzer`` instances as Arrow ``large_string`` columns - tagged with extension name ``"spikeinterface.sorting_analyzer"``. Unlike - ``BaseRecording`` / ``BaseSorting``, ``SortingAnalyzer`` is not a - ``BaseExtractor`` subclass and has no ``to_dict()`` round-trip — it is - exclusively folder-backed (``binary_folder`` or ``zarr``). The stored value - is a JSON object ``{"folder": "", "format": ""}`` - derived from ``analyzer.folder`` and ``analyzer.format``. - - The analyzer must be saved to disk before passing it to orcapod (i.e. - ``analyzer.folder`` must not be ``None``). In-memory analyzers raise - ``ValueError`` with save instructions. Loading reconstructs the analyzer - via ``SortingAnalyzer.load(folder)`` with automatic format detection. - - Example: - >>> import tempfile, numpy as np - >>> import spikeinterface.core as si_core - >>> from orcapod.extension_types.spikeinterface_types import LogicalSISortingAnalyzer - >>> lt = LogicalSISortingAnalyzer() - >>> with tempfile.TemporaryDirectory() as tmp: - ... recording = si_core.NumpyRecording( - ... [np.zeros((200, 4), dtype="float32")], 30000 - ... ) - ... sorting = si_core.NumpySorting.from_unit_dict( - ... {0: np.array([0, 50, 100])}, sampling_frequency=30000 - ... ) - ... analyzer = si_core.SortingAnalyzer.create( - ... sorting, recording, format="binary_folder", folder=tmp + "/analyzer" - ... ) - ... storage = lt.python_to_storage(analyzer) - ... recovered = lt.storage_to_python(storage) - ... recovered.sorting.get_unit_ids().tolist() == analyzer.sorting.get_unit_ids().tolist() - True - """ - - _arrow_ext_class = make_arrow_extension_type("spikeinterface.sorting_analyzer", pa.large_string()) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type("spikeinterface.sorting_analyzer", pa.large_string()) - _polars_ext: pl.BaseExtension | None = None - - logical_type_name: str = "spikeinterface.sorting_analyzer" - python_type: type = SortingAnalyzer - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type for ``SortingAnalyzer``. - - Returns: - A ``pa.ExtensionType`` with extension name - ``"spikeinterface.sorting_analyzer"`` and storage type - ``pa.large_string()``. - """ - if LogicalSISortingAnalyzer._arrow_ext is None: - LogicalSISortingAnalyzer._arrow_ext = LogicalSISortingAnalyzer._arrow_ext_class() - return LogicalSISortingAnalyzer._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for ``SortingAnalyzer``. - - Returns: - A ``pl.BaseExtension`` registered under - ``"spikeinterface.sorting_analyzer"``. - """ - if LogicalSISortingAnalyzer._polars_ext is None: - LogicalSISortingAnalyzer._polars_ext = LogicalSISortingAnalyzer._polars_ext_class() - return LogicalSISortingAnalyzer._polars_ext - - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None = None - ) -> str: - """Serialise a ``SortingAnalyzer`` to its JSON storage representation. - - Args: - value: A ``SortingAnalyzer`` instance with a non-``None`` ``folder`` - attribute (i.e. saved via ``save_as()`` or created with - ``format="binary_folder"`` or ``format="zarr"``). - converter: Ignored. Present for protocol conformance. - - Returns: - A JSON string ``{"folder": "", "format": ""}`` where - ```` is ``str(analyzer.folder)`` and ```` is - ``analyzer.format`` (``"binary_folder"`` or ``"zarr"``). - - Raises: - ValueError: If ``analyzer.folder`` is ``None`` (in-memory analyzer - that has not been saved to disk). - """ - if value.folder is None: - raise ValueError( - "SortingAnalyzer has no folder (in-memory). " - "Call analyzer.save_as(format='binary_folder'|'zarr', folder=) " - "first, then pass the saved analyzer to the pod." - ) - return json.dumps({"folder": str(value.folder), "format": value.format}) - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None = None - ) -> SortingAnalyzer: - """Reconstruct a ``SortingAnalyzer`` from its JSON storage string. - - Args: - storage_value: A JSON string as stored in Arrow, with keys - ``"folder"`` and ``"format"``. - converter: Ignored. Present for protocol conformance. - - Returns: - A ``SortingAnalyzer`` instance loaded via - ``SortingAnalyzer.load(folder)``. The format is auto-detected from - the folder suffix (``".zarr"`` → zarr, otherwise binary_folder). - - Raises: - ValueError: If ``storage_value`` is not valid JSON. - FileNotFoundError: If the analyzer folder no longer exists - (raised by SpikeInterface, propagated as-is). - """ - try: - data = json.loads(storage_value) - except (json.JSONDecodeError, TypeError) as exc: - raise ValueError( - f"LogicalSISortingAnalyzer: cannot deserialise storage value " - f"{storage_value!r}; expected a JSON string." - ) from exc - return SortingAnalyzer.load(data["folder"]) -``` - -- [ ] **Step 5: Run importable test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_sorting_analyzer_importable -v -``` - -Expected: `PASSED` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py \ - tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add LogicalSISortingAnalyzer class (ITL-469)" -``` - ---- - -## Task 2: `python_to_storage` / `storage_to_python` tests - -**Files:** -- Test: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Write in-memory guard test + round-trip tests** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_in_memory_analyzer_raises(): - """In-memory SortingAnalyzer (folder=None) raises ValueError with clear save instructions.""" - si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISortingAnalyzer - - rng = np.random.default_rng(0) - traces = rng.standard_normal((100, 2)).astype("float32") - recording = si_core.NumpyRecording([traces], sampling_frequency=30_000) - sorting = si_core.NumpySorting.from_unit_dict( - {0: np.array([10, 50, 90])}, sampling_frequency=30_000 - ) - analyzer = si_core.SortingAnalyzer.create(sorting, recording, format="memory") - - lt = LogicalSISortingAnalyzer() - with pytest.raises(ValueError, match="in-memory"): - lt.python_to_storage(analyzer) - - -def test_binary_folder_analyzer_round_trip(tmp_path): - """binary_folder-backed SortingAnalyzer round-trips through python_to_storage / storage_to_python.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISortingAnalyzer - - analyzer = _make_sorting_analyzer(tmp_path, "binary_folder") - lt = LogicalSISortingAnalyzer() - - storage = lt.python_to_storage(analyzer) - assert isinstance(storage, str) - data = json.loads(storage) - assert "folder" in data - assert data["format"] == "binary_folder" - - recovered = lt.storage_to_python(storage) - np.testing.assert_array_equal( - analyzer.sorting.get_unit_spike_train(unit_id=0, segment_index=0), - recovered.sorting.get_unit_spike_train(unit_id=0, segment_index=0), - ) - - -def test_zarr_analyzer_round_trip(tmp_path): - """Zarr-backed SortingAnalyzer round-trips through python_to_storage / storage_to_python.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import LogicalSISortingAnalyzer - - analyzer = _make_sorting_analyzer(tmp_path, "zarr") - lt = LogicalSISortingAnalyzer() - - storage = lt.python_to_storage(analyzer) - assert isinstance(storage, str) - data = json.loads(storage) - assert "folder" in data - assert data["format"] == "zarr" - - recovered = lt.storage_to_python(storage) - np.testing.assert_array_equal( - analyzer.sorting.get_unit_spike_train(unit_id=0, segment_index=0), - recovered.sorting.get_unit_spike_train(unit_id=0, segment_index=0), - ) -``` - -- [ ] **Step 2: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "test_in_memory_analyzer_raises or test_binary_folder_analyzer_round_trip or test_zarr_analyzer_round_trip" -v -``` - -Expected: all 3 `PASSED` (the implementation was already added in Task 1 Step 4). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_extension_types/test_spikeinterface_types.py -git commit -m "test(extension_types): add LogicalSISortingAnalyzer serialization tests (ITL-469)" -``` - ---- - -## Task 3: `SISortingAnalyzerHandler` + handler tests - -**Files:** -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` -- Test: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Write the failing handler tests** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -# ── SISortingAnalyzerHandler tests ──────────────────────────────────────────── - -def test_si_sorting_analyzer_handler_hash_stability(tmp_path): - """Same SortingAnalyzer produces identical ContentHash across two calls.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SISortingAnalyzerHandler - from orcapod.types import ContentHash - - analyzer = _make_sorting_analyzer(tmp_path, "binary_folder") - handler = SISortingAnalyzerHandler() - - h1 = handler.handle(analyzer, hasher=None) - h2 = handler.handle(analyzer, hasher=None) - - assert isinstance(h1, ContentHash) - assert h1 == h2 - - -def test_si_sorting_analyzer_handler_hash_changes_with_path(tmp_path): - """SortingAnalyzers at different folder paths produce different ContentHash values.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SISortingAnalyzerHandler - - analyzer_a = _make_sorting_analyzer(tmp_path / "a", "binary_folder") - analyzer_b = _make_sorting_analyzer(tmp_path / "b", "binary_folder") - - handler = SISortingAnalyzerHandler() - assert handler.handle(analyzer_a, hasher=None) != handler.handle(analyzer_b, hasher=None) - - -def test_si_sorting_analyzer_handler_in_memory_raises(): - """``SISortingAnalyzerHandler`` raises ValueError for in-memory analyzers.""" - si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import SISortingAnalyzerHandler - - rng = np.random.default_rng(0) - traces = rng.standard_normal((100, 2)).astype("float32") - recording = si_core.NumpyRecording([traces], sampling_frequency=30_000) - sorting = si_core.NumpySorting.from_unit_dict( - {0: np.array([10, 50, 90])}, sampling_frequency=30_000 - ) - analyzer = si_core.SortingAnalyzer.create(sorting, recording, format="memory") - - handler = SISortingAnalyzerHandler() - with pytest.raises(ValueError, match="in-memory"): - handler.handle(analyzer, hasher=None) -``` - -- [ ] **Step 2: Run handler tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "test_si_sorting_analyzer_handler_hash_stability or test_si_sorting_analyzer_handler_hash_changes_with_path or test_si_sorting_analyzer_handler_in_memory_raises" -v -``` - -Expected: all 3 `FAILED` — `ImportError` (class does not exist yet). - -- [ ] **Step 3: Add `SISortingAnalyzerHandler` class** - -In `src/orcapod/extension_types/spikeinterface_types.py`, append after `LogicalSISortingAnalyzer` and before `def register_spikeinterface_types`: - -```python -class SISortingAnalyzerHandler: - """Semantic hash handler for ``spikeinterface.core.SortingAnalyzer``. - - Computes a SHA-256 ``ContentHash`` of the folder path string - (``str(analyzer.folder).encode()``). This is phase 1 — path-string - hashing. Content hashing of the folder contents is deferred to ITL-476. - - The ``hasher`` argument is accepted for protocol conformance but not used — - hashing is done directly via ``hashlib.sha256`` to avoid overhead. - """ - - def handle(self, obj: Any, hasher: SemanticHasherProtocol | None) -> ContentHash: - """Return a SHA-256 ``ContentHash`` of the analyzer's folder path string. - - Args: - obj: A ``SortingAnalyzer`` instance. - hasher: Accepted for protocol conformance; not used. - - Returns: - A ``ContentHash`` with ``method="sha256"`` and digest equal to the - SHA-256 of ``str(analyzer.folder).encode()``. - - Raises: - ValueError: If ``analyzer.folder`` is ``None`` (in-memory analyzer). - """ - if obj.folder is None: - raise ValueError( - "Cannot hash in-memory SortingAnalyzer (folder is None). " - "Call save_as() first." - ) - folder_bytes = str(obj.folder).encode() - logger.debug("SISortingAnalyzerHandler: hashing folder path %r", str(obj.folder)) - return ContentHash( - method="sha256", - digest=hashlib.sha256(folder_bytes).digest(), - ) -``` - -- [ ] **Step 4: Run handler tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py \ - -k "test_si_sorting_analyzer_handler_hash_stability or test_si_sorting_analyzer_handler_hash_changes_with_path or test_si_sorting_analyzer_handler_in_memory_raises" -v -``` - -Expected: all 3 `PASSED` - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py \ - tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): add SISortingAnalyzerHandler (ITL-469)" -``` - ---- - -## Task 4: Wire into `register_spikeinterface_types()` + registration test - -**Files:** -- Modify: `src/orcapod/extension_types/spikeinterface_types.py` -- Test: `tests/test_extension_types/test_spikeinterface_types.py` - -- [ ] **Step 1: Write the failing registration test** - -Append to `tests/test_extension_types/test_spikeinterface_types.py`: - -```python -def test_register_spikeinterface_types_includes_sorting_analyzer(tmp_path): - """``register_spikeinterface_types()`` wires ``LogicalSISortingAnalyzer`` and - ``SISortingAnalyzerHandler`` into the default context so they are found by type lookup.""" - pytest.importorskip("spikeinterface", reason="spikeinterface not installed") - from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - from orcapod.contexts import get_default_context - - register_spikeinterface_types() - ctx = get_default_context() - - analyzer = _make_sorting_analyzer(tmp_path, "binary_folder") - - # LogicalType registered: type_converter can find it - arrow_type = ctx.type_converter.python_type_to_arrow_type(type(analyzer)) - assert arrow_type.extension_name == "spikeinterface.sorting_analyzer" - - # Handler registered: semantic_hasher can find it - from orcapod.types import ContentHash - handler = ctx.semantic_hasher.type_handler_registry.get_handler(analyzer) - assert handler is not None - result = handler.handle(analyzer, hasher=None) - assert isinstance(result, ContentHash) - - # Full round-trip through type_converter - storage = ctx.type_converter.python_to_storage(analyzer, type(analyzer)) - recovered = ctx.type_converter.storage_to_python(storage, type(analyzer)) - np.testing.assert_array_equal( - analyzer.sorting.get_unit_spike_train(unit_id=0, segment_index=0), - recovered.sorting.get_unit_spike_train(unit_id=0, segment_index=0), - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_register_spikeinterface_types_includes_sorting_analyzer -v -``` - -Expected: `FAILED` — `AssertionError` because `register_spikeinterface_types` does not yet register `LogicalSISortingAnalyzer`. - -- [ ] **Step 3: Add 4th block to `register_spikeinterface_types()`** - -In `src/orcapod/extension_types/spikeinterface_types.py`, find `def register_spikeinterface_types`. Replace the last 2 lines (the Motion handler registration and the closing of the function body) with: - -Current end of function (lines ~681–683): -```python - # Handler registration silently replaces an existing entry, so always safe. - context.semantic_hasher.type_handler_registry.register(Motion, SIMotionHandler()) -``` - -Replace with: -```python - # Handler registration silently replaces an existing entry, so always safe. - context.semantic_hasher.type_handler_registry.register(Motion, SIMotionHandler()) - - # --- SortingAnalyzer --- - lt_sorting_analyzer = LogicalSISortingAnalyzer() - try: - context.type_converter.register_logical_type(lt_sorting_analyzer) - except ValueError as exc: - # A different LogicalSISortingAnalyzer instance is already registered (e.g. - # auto-registered from v0.1.json at context creation time). That is - # fine — both instances are equivalent. Any other ValueError propagates. - if "already bound to" not in str(exc): - raise - logger.debug( - "register_spikeinterface_types: LogicalSISortingAnalyzer already registered, skipping" - ) - else: - logger.debug("register_spikeinterface_types: registered LogicalSISortingAnalyzer") - - # Handler registration silently replaces an existing entry, so always safe. - context.semantic_hasher.type_handler_registry.register( - SortingAnalyzer, SISortingAnalyzerHandler() - ) -``` - -Also update the docstring of `register_spikeinterface_types` — change the first line: - -```python - """Register SpikeInterface LogicalTypes into an orcapod ``DataContext``. - - Registers ``LogicalSIRecording`` / ``SIRecordingHandler`` (ITL-459), - ``LogicalSISorting`` / ``SISortingHandler`` (ITL-468), - ``LogicalSIMotion`` / ``SIMotionHandler`` (ITL-470), and - ``LogicalSISortingAnalyzer`` / ``SISortingAnalyzerHandler`` (ITL-469). -``` - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py::test_register_spikeinterface_types_includes_sorting_analyzer -v -``` - -Expected: `PASSED` - -- [ ] **Step 5: Run all SI tests to check no regressions** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v -``` - -Expected: all tests `PASSED` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/spikeinterface_types.py \ - tests/test_extension_types/test_spikeinterface_types.py -git commit -m "feat(extension_types): wire LogicalSISortingAnalyzer into register_spikeinterface_types (ITL-469)" -``` - ---- - -## Task 5: Wire into `__init__.py` and `v0.1.json` - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Update `__init__.py` conditional imports** - -In `src/orcapod/extension_types/__init__.py`, change the SI try-block (lines 34–44): - -```python -# ITL-459, ITL-468, ITL-470, ITL-469 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import ( - LogicalSIRecording, - LogicalSISorting, - LogicalSIMotion, - LogicalSISortingAnalyzer, - SIMotionHandler, - SISortingAnalyzerHandler, - register_spikeinterface_types, - ) - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -Change the `__all__` SI conditional block (lines 73–82): - -```python - # ITL-459, ITL-468, ITL-470, ITL-469 (conditional — only present when spikeinterface is installed) - *( - [ - "LogicalSIRecording", - "LogicalSISorting", - "LogicalSIMotion", - "LogicalSISortingAnalyzer", - "SIMotionHandler", - "SISortingAnalyzerHandler", - "register_spikeinterface_types", - ] if _SI_AVAILABLE else [] - ), -``` - -- [ ] **Step 2: Add `LogicalSISortingAnalyzer` entry to `v0.1.json` logical_types** - -In `src/orcapod/contexts/data/v0.1.json`, in the `"logical_types"` array, add after the `LogicalSIMotion` entry (after line 62): - -```json - { - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSISortingAnalyzer", - "_config": {}, - "_optional": true - }, -``` - -- [ ] **Step 3: Add `SISortingAnalyzerHandler` entry to `v0.1.json` handlers** - -In `src/orcapod/contexts/data/v0.1.json`, in the `"handlers"` array, add after the `SIMotionHandler` entry (after line 122): - -```json - [{"_type": "spikeinterface.core.SortingAnalyzer", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SISortingAnalyzerHandler", "_config": {}, "_optional": true}], -``` - -- [ ] **Step 4: Add changelog entry to `v0.1.json`** - -In `src/orcapod/contexts/data/v0.1.json`, append to the `"changelog"` array: - -```json - "Added spikeinterface.core.SortingAnalyzer as a native value type via LogicalSISortingAnalyzer (large_string/JSON path reference {folder, format}) and SISortingAnalyzerHandler (SHA-256 of folder path string, phase 1); auto-registered when spikeinterface is installed via _optional entries in v0.1.json (ITL-469)" -``` - -- [ ] **Step 5: Update module docstring in `spikeinterface_types.py`** - -In `src/orcapod/extension_types/spikeinterface_types.py`, change the first line of the module docstring from: - -```python -"""SpikeInterface LogicalTypes and handlers for orcapod (ITL-459, ITL-468, ITL-470). -``` - -to: - -```python -"""SpikeInterface LogicalTypes and handlers for orcapod (ITL-459, ITL-468, ITL-470, ITL-469). -``` - -Also add a paragraph after the Motion description: - -``` -``LogicalSISortingAnalyzer`` maps ``spikeinterface.core.SortingAnalyzer`` ↔ Arrow -``large_string`` using a JSON path-reference object -``{"folder": "", "format": ""}`` derived from -``analyzer.folder`` and ``analyzer.format``. Unlike Recording/Sorting, there is -no ``to_dict()`` round-trip — the analyzer is exclusively folder-backed. -``SISortingAnalyzerHandler`` hashes the folder path string via SHA-256 (phase 1). -``` - -- [ ] **Step 6: Run the full SI test suite** - -```bash -uv run pytest tests/test_extension_types/test_spikeinterface_types.py -v -``` - -Expected: all tests `PASSED` - -- [ ] **Step 7: Run the full test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests `PASSED` (no regressions) - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py \ - src/orcapod/contexts/data/v0.1.json \ - src/orcapod/extension_types/spikeinterface_types.py -git commit -m "feat(extension_types): support SpikeInterface SortingAnalyzer objects (ITL-469)" -``` diff --git a/superpowers/plans/2026-07-05-itl-500-populate-hash-cache-ergonomics.md b/superpowers/plans/2026-07-05-itl-500-populate-hash-cache-ergonomics.md deleted file mode 100644 index b093feee8..000000000 --- a/superpowers/plans/2026-07-05-itl-500-populate-hash-cache-ergonomics.md +++ /dev/null @@ -1,1496 +0,0 @@ -# populate_hash_cache Ergonomics Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `progress_callback`, `dry_run`, `force`, and `total_bytes_cached` to `populate_hash_cache` using a clean visitor/accumulator split. - -**Architecture:** A private `_Accumulator` owns a mutable `_Stats` object and the optional callback; two visitor classes (`_HashVisitor`, `_DryRunVisitor`) replace the old `_hash_one` function; `populate_hash_cache` constructs the right visitor and delegates all per-file logic to it. The `pending` set becomes a `dict[Future, Path]` so callbacks receive the file path. - -**Tech Stack:** Python 3.12, `concurrent.futures`, `uv run pytest` - -**Spec:** `superpowers/specs/2026-07-05-itl-500-populate-hash-cache-ergonomics-design.md` - -**Branch:** `eywalker/itl-500-improve-populate_hash_cache-ergonomics-progress-callback-dry` - ---- - -## Task 0: Create and check out the feature branch - -**Files:** none - -- [ ] **Step 0.1: Create the branch from `main`** - -```bash -git checkout main -git checkout -b eywalker/itl-500-improve-populate_hash_cache-ergonomics-progress-callback-dry -git branch --show-current -``` - -Expected output: `eywalker/itl-500-improve-populate_hash_cache-ergonomics-progress-callback-dry` - ---- - -## Task 1: Add `total_bytes_cached` to `CachePopulationStats` - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` -- Modify: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 1.1: Write failing tests** - -Add to `tests/test_hashing/test_cache_population.py`: - -```python -# Update TestCachePopulationStats.test_instantiation — add total_bytes_cached=50: -class TestCachePopulationStats: - def test_instantiation(self): - from orcapod.hashing.cache_population import CachePopulationStats - - stats = CachePopulationStats( - hashed=1, - already_cached=2, - skipped_small=3, - errors=0, - total_bytes_hashed=100, - total_bytes_cached=50, - total_duration=1.0, - avg_hashing_speed=100.0, - ) - assert stats.hashed == 1 - assert stats.already_cached == 2 - assert stats.skipped_small == 3 - assert stats.errors == 0 - assert stats.total_bytes_hashed == 100 - assert stats.total_bytes_cached == 50 - assert stats.total_duration == 1.0 - assert stats.avg_hashing_speed == 100.0 - - def test_is_frozen(self): - from orcapod.hashing.cache_population import CachePopulationStats - - stats = CachePopulationStats( - hashed=0, - already_cached=0, - skipped_small=0, - errors=0, - total_bytes_hashed=0, - total_bytes_cached=0, - total_duration=0.0, - avg_hashing_speed=0.0, - ) - with pytest.raises((dataclasses.FrozenInstanceError, AttributeError)): - stats.hashed = 1 # type: ignore[misc] -``` - -Add new class at end of file: - -```python -class TestCachedBytes: - def test_total_bytes_cached_zero_on_first_run(self, tmp_path): - """First run has nothing cached yet — total_bytes_cached must be zero.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - assert stats.total_bytes_cached == 0 - - def test_total_bytes_cached_on_second_run(self, tmp_path): - """Second run finds the file cached — total_bytes_cached equals file size.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - second = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert second.total_bytes_cached == 20 - assert second.total_bytes_hashed == 0 - - def test_total_bytes_cached_multiple_files(self, tmp_path): - """total_bytes_cached sums across all already-cached files.""" - db = tmp_path / "cache.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 30) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - second = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert second.total_bytes_cached == 50 - assert second.total_bytes_hashed == 0 -``` - -Also update `TestCacheHitMiss.test_cache_hit_skips_rehash` to assert `total_bytes_cached`: - -```python -def test_cache_hit_skips_rehash(self, tmp_path): - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - first = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - second = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - assert first.hashed == 1 - assert first.total_bytes_cached == 0 - assert second.already_cached == 1 - assert second.hashed == 0 - assert second.total_bytes_cached == 20 -``` - -- [ ] **Step 1.2: Run tests to confirm they fail** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -15 -``` - -Expected: failures on `test_instantiation`, `test_is_frozen`, `test_cache_hit_skips_rehash`, and all `TestCachedBytes` tests (missing `total_bytes_cached` field). - -- [ ] **Step 1.3: Add `total_bytes_cached` to `CachePopulationStats`** - -In `src/orcapod/hashing/cache_population.py`, update the dataclass: - -```python -@dataclass(frozen=True) -class CachePopulationStats: - """Statistics returned by ``populate_hash_cache()``. - - Attributes: - hashed: Files newly hashed and written to the cache. - already_cached: Files whose hash was already in the cache (rehash skipped). - skipped_small: Files skipped because their size was below ``min_size_bytes``. - errors: Files or directories that raised an exception during traversal or hashing - (logged as warnings). - total_bytes_hashed: Sum of bytes across all newly hashed files. - total_bytes_cached: Sum of bytes across all already-cached files. - total_duration: Wall-clock duration of the full run in seconds. - avg_hashing_speed: Hashing throughput in bytes per second - (``total_bytes_hashed / total_duration``; ``0.0`` if ``total_duration`` is zero). - """ - - hashed: int - already_cached: int - skipped_small: int - errors: int - total_bytes_hashed: int - total_bytes_cached: int - total_duration: float - avg_hashing_speed: float -``` - -- [ ] **Step 1.4: Update `_hash_one` to return file size for cached hits** - -In `_hash_one`, change the cached return from `("cached", 0)` to `("cached", file_stat.st_size)`: - -```python -def _hash_one( - resolved: UPath, - file_stat: os.stat_result, - cacher: SqliteHashCacher, - hasher: FileHasher, -) -> tuple[Literal["hashed", "cached", "error"], int]: - """Hash and cache a single qualifying file, returning a (category, bytes) result pair. - ... - """ - try: - key = FileHashKey(resolved, file_stat.st_mtime_ns, file_stat.st_size) - if cacher.get(key) is not None: - return ("cached", file_stat.st_size) - content_hash = hasher.hash_file(resolved) - cacher.put(key, content_hash) - return ("hashed", file_stat.st_size) - except Exception: - logger.warning("Failed to process file %s", resolved, exc_info=True) - return ("error", 0) -``` - -- [ ] **Step 1.5: Add `already_cached_bytes` accumulator and `total_bytes_cached` to the return in `populate_hash_cache`** - -In `populate_hash_cache`, inside the `with SqliteHashCacher(_db_path) as cacher:` block, add `already_cached_bytes = 0` alongside the other counter variables: - -```python - error_count = 0 - hashed = 0 - already_cached = 0 - skipped_small = 0 - total_bytes_hashed = 0 - already_cached_bytes = 0 - start = time.monotonic() -``` - -In both drain locations (the backpressure drain and the final drain), update the `"cached"` branch to accumulate bytes. Replace both occurrences of: - -```python - elif cat == "cached": - already_cached += 1 -``` - -With: - -```python - elif cat == "cached": - already_cached += 1 - already_cached_bytes += nbytes -``` - -Update the `return` statement at the bottom: - -```python - return CachePopulationStats( - hashed=hashed, - already_cached=already_cached, - skipped_small=skipped_small, - errors=error_count, - total_bytes_hashed=total_bytes_hashed, - total_bytes_cached=already_cached_bytes, - total_duration=duration, - avg_hashing_speed=speed, - ) -``` - -- [ ] **Step 1.6: Run tests to confirm they pass** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -10 -``` - -Expected: `25 passed` (22 original + 3 new `TestCachedBytes` tests). - -- [ ] **Step 1.7: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): add total_bytes_cached to CachePopulationStats (ITL-500)" -``` - ---- - -## Task 2: Introduce `_Stats`, `_Accumulator`, `_HashVisitor` — pure refactoring - -No new behavior. All 25 tests must still pass after this task. - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` - -- [ ] **Step 2.1: Run existing tests to establish baseline** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -5 -``` - -Expected: `25 passed`. - -- [ ] **Step 2.2: Add type aliases and `_Stats` dataclass** - -After the `CachePopulationStats` dataclass and before `_hash_one`, add: - -```python -FileOutcome = Literal["hashed", "cached", "would_hash", "error"] - - -@dataclass -class _Stats: - """Private mutable accumulator for per-run counters. - - Replaces the five loose local variables previously used inside - ``populate_hash_cache``. - """ - - hashed: int = 0 - already_cached: int = 0 - skipped_small: int = 0 - errors: int = 0 - total_bytes_hashed: int = 0 - total_bytes_cached: int = 0 - start: float = field(default_factory=time.monotonic) - - def snapshot(self) -> CachePopulationStats: - """Return a frozen ``CachePopulationStats`` snapshot of current totals.""" - duration = time.monotonic() - self.start - return CachePopulationStats( - hashed=self.hashed, - already_cached=self.already_cached, - skipped_small=self.skipped_small, - errors=self.errors, - total_bytes_hashed=self.total_bytes_hashed, - total_bytes_cached=self.total_bytes_cached, - total_duration=duration, - avg_hashing_speed=( - self.total_bytes_hashed / duration if duration > 0 else 0.0 - ), - ) -``` - -Also add `field` to the dataclasses import at the top of the file: - -```python -from dataclasses import dataclass, field -``` - -- [ ] **Step 2.3: Add `_Accumulator` class** - -After `_Stats`, add: - -```python -from typing import Callable -from pathlib import Path - - -class _Accumulator: - """Owns a ``_Stats`` object and an optional progress callback. - - All counter updates flow through this class so there is one place - where snapshots are created and the callback is fired. - """ - - def __init__(self, callback: "ProgressCallback | None" = None) -> None: - self._stats = _Stats() - self._callback = callback - - def record(self, path: Path, outcome: FileOutcome, nbytes: int) -> None: - """Update counters for a qualifying file and fire the callback if set. - - Args: - path: Resolved file path that was just processed. - outcome: What happened to the file. - nbytes: File size in bytes (for hashed/would_hash/cached outcomes); - ``0`` for errors. - """ - if outcome in ("hashed", "would_hash"): - self._stats.hashed += 1 - self._stats.total_bytes_hashed += nbytes - elif outcome == "cached": - self._stats.already_cached += 1 - self._stats.total_bytes_cached += nbytes - else: # "error" - self._stats.errors += 1 - if self._callback is not None: - self._callback(path, outcome, self._stats.snapshot()) - - def record_skipped_small(self) -> None: - """Increment ``skipped_small`` without firing the callback.""" - self._stats.skipped_small += 1 - - def record_directory_error(self) -> None: - """Increment ``errors`` for a directory access failure without firing the callback.""" - self._stats.errors += 1 - - def finalize(self) -> CachePopulationStats: - """Return the final frozen ``CachePopulationStats`` snapshot.""" - return self._stats.snapshot() -``` - -Note: `ProgressCallback` is defined in Task 5; use a forward reference string for now. - -- [ ] **Step 2.4: Add `_HashVisitor` and remove `_hash_one`** - -Replace the entire `_hash_one` function with `_HashVisitor`: - -```python -class _HashVisitor: - """Per-file visitor that checks the cache and hashes on miss. - - Callable with signature ``(resolved, file_stat) -> (FileOutcome, int)``. - Thread-safe: ``SqliteHashCacher`` uses ``threading.local()`` connections - and ``FileHasher`` is stateless. - - Args: - cacher: Shared ``SqliteHashCacher`` instance. - hasher: Shared ``FileHasher`` instance. - force: If ``True``, skip the cache-get and always re-hash. - """ - - def __init__( - self, - cacher: SqliteHashCacher, - hasher: FileHasher, - *, - force: bool = False, - ) -> None: - self._cacher = cacher - self._hasher = hasher - self._force = force - - def __call__( - self, resolved: UPath, file_stat: os.stat_result - ) -> tuple[FileOutcome, int]: - try: - key = FileHashKey(resolved, file_stat.st_mtime_ns, file_stat.st_size) - if not self._force and self._cacher.get(key) is not None: - return ("cached", file_stat.st_size) - content_hash = self._hasher.hash_file(resolved) - self._cacher.put(key, content_hash) - return ("hashed", file_stat.st_size) - except Exception: - logger.warning("Failed to process file %s", resolved, exc_info=True) - return ("error", 0) -``` - -- [ ] **Step 2.5: Refactor `populate_hash_cache` to use `_Accumulator`, `_HashVisitor`, and `dict[Future, Path]`** - -Replace the body of `populate_hash_cache` (everything inside `with ThreadPoolExecutor` and `with SqliteHashCacher`). The new body: - -```python - root = UPath(path) - _db_path: Path | None = Path(db_path) if db_path is not None else None - hasher = FileHasher(algorithm=algorithm, buffer_size=buffer_size) - - with SqliteHashCacher(_db_path) as cacher: - _db_resolved = cacher.db_path.resolve() - _db_str = str(_db_resolved) - _excluded: frozenset[UPath] = frozenset( - [ - UPath(_db_resolved), - UPath(_db_str + "-wal"), - UPath(_db_str + "-shm"), - UPath(_db_str + "-journal"), - ] - ) - - visitor = _HashVisitor(cacher, hasher) - accumulator = _Accumulator() - _max_pending = max_workers * _PENDING_FACTOR - pending: dict[Future[tuple[FileOutcome, int]], UPath] = {} - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - stack: list[UPath] = [root] - while stack: - current = stack.pop() - try: - entries = list(current.iterdir()) - except OSError: - logger.warning( - "Cannot access directory %s", current, exc_info=True - ) - accumulator.record_directory_error() - continue - - for entry in entries: - if entry.is_symlink(): - continue - if entry.is_dir(): - stack.append(entry) - continue - if not entry.is_file(): - continue - - try: - resolved = entry.resolve() - if resolved in _excluded: - continue - file_stat = resolved.stat() - except OSError: - logger.warning( - "Cannot stat file %s", entry, exc_info=True - ) - accumulator.record(entry, "error", 0) - continue - - if file_stat.st_size < min_size_bytes: - accumulator.record_skipped_small() - continue - - if len(pending) >= _max_pending: - done_futures, _ = wait(pending.keys(), return_when=FIRST_COMPLETED) - for fut in done_futures: - p = pending.pop(fut) - outcome, nbytes = fut.result() - accumulator.record(p, outcome, nbytes) - - future = executor.submit(visitor, resolved, file_stat) - pending[future] = resolved - - for fut in as_completed(pending): - p = pending[fut] - outcome, nbytes = fut.result() - accumulator.record(p, outcome, nbytes) - - return accumulator.finalize() -``` - -Also remove the old local variable declarations (`error_count = 0`, `hashed = 0`, etc.) and the old `duration`/`speed`/`return` at the bottom — all now inside `accumulator.finalize()`. - -- [ ] **Step 2.6: Run all tests to confirm refactoring preserves behavior** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -10 -``` - -Expected: `25 passed` — no regressions. - -- [ ] **Step 2.7: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py -git commit -m "refactor(hashing): introduce _Stats, _Accumulator, _HashVisitor (ITL-500)" -``` - ---- - -## Task 3: Add `force` parameter - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` -- Modify: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 3.1: Write failing tests** - -Add to `tests/test_hashing/test_cache_population.py`: - -```python -class TestForce: - def test_force_rehashes_cached_file(self, tmp_path): - """force=True re-hashes a file even if already in the cache.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - first = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - assert first.hashed == 1 - - second = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db, force=True - ) - assert second.hashed == 1 - assert second.already_cached == 0 - - def test_force_false_skips_cached_file(self, tmp_path): - """force=False (default) does not re-hash an already-cached file.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - second = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db, force=False - ) - assert second.already_cached == 1 - assert second.hashed == 0 - - def test_force_bytes_hashed(self, tmp_path): - """force=True: total_bytes_hashed counts re-hashed bytes; total_bytes_cached is zero.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - second = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db, force=True - ) - assert second.total_bytes_hashed == 20 - assert second.total_bytes_cached == 0 -``` - -- [ ] **Step 3.2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestForce -q 2>&1 | tail -10 -``` - -Expected: `TypeError: populate_hash_cache() got an unexpected keyword argument 'force'` - -- [ ] **Step 3.3: Add `force` parameter to `populate_hash_cache` and pass it to `_HashVisitor`** - -Add `force: bool = False` to the signature: - -```python -def populate_hash_cache( - path: PathLike | UPath, - *, - min_size_bytes: int = _DEFAULT_MIN_SIZE_BYTES, - db_path: UPath | PathLike | None = None, - algorithm: str = "sha256", - buffer_size: int = 65536, - max_workers: int = _DEFAULT_MAX_WORKERS, - force: bool = False, -) -> CachePopulationStats: -``` - -Update the docstring to document `force`: - -``` - force: If ``True``, re-hash files even if they already have a cache entry. - Defaults to ``False``. -``` - -Pass `force` to `_HashVisitor` where the visitor is constructed: - -```python - visitor = _HashVisitor(cacher, hasher, force=force) -``` - -- [ ] **Step 3.4: Run tests to confirm they pass** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -10 -``` - -Expected: `28 passed`. - -- [ ] **Step 3.5: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): add force parameter to populate_hash_cache (ITL-500)" -``` - ---- - -## Task 4: Add `_DryRunVisitor` and `dry_run` parameter - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` -- Modify: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 4.1: Write failing tests** - -Add to `tests/test_hashing/test_cache_population.py`: - -```python -class TestVisitors: - def test_dry_run_visitor_miss_returns_would_hash(self, tmp_path): - """_DryRunVisitor returns would_hash for a file not in the cache.""" - import os - from orcapod.hashing.cache_population import _DryRunVisitor - from orcapod.hashing.hash_cachers import SqliteHashCacher - - db = tmp_path / "cache.db" - f = _write(tmp_path, "f.bin", 20) - resolved = f.resolve() - file_stat = resolved.stat() - - with SqliteHashCacher(db) as cacher: - visitor = _DryRunVisitor(cacher) - outcome, nbytes = visitor(resolved, file_stat) - - assert outcome == "would_hash" - assert nbytes == 20 - - def test_dry_run_visitor_hit_returns_cached(self, tmp_path): - """_DryRunVisitor returns cached for a file already in the cache.""" - import os - from orcapod.hashing.cache_population import _DryRunVisitor, populate_hash_cache - from orcapod.hashing.hash_cachers import SqliteHashCacher - - db = tmp_path / "cache.db" - f = _write(tmp_path, "f.bin", 20) - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - resolved = f.resolve() - file_stat = resolved.stat() - - with SqliteHashCacher(db) as cacher: - visitor = _DryRunVisitor(cacher) - outcome, nbytes = visitor(resolved, file_stat) - - assert outcome == "cached" - assert nbytes == 20 - - def test_dry_run_visitor_force_always_would_hash(self, tmp_path): - """_DryRunVisitor with force=True always returns would_hash regardless of cache.""" - from orcapod.hashing.cache_population import _DryRunVisitor, populate_hash_cache - from orcapod.hashing.hash_cachers import SqliteHashCacher - - db = tmp_path / "cache.db" - f = _write(tmp_path, "f.bin", 20) - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - resolved = f.resolve() - file_stat = resolved.stat() - - with SqliteHashCacher(db) as cacher: - visitor = _DryRunVisitor(cacher, force=True) - outcome, nbytes = visitor(resolved, file_stat) - - assert outcome == "would_hash" - assert nbytes == 20 - - -class TestDryRun: - def test_dry_run_no_cache_writes(self, tmp_path): - """dry_run=True must not write any entries to the cache.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db, dry_run=True) - - # Second run with dry_run=False should hash (not find cache entries). - second = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - assert second.hashed == 1 - assert second.already_cached == 0 - - def test_dry_run_hashed_equals_would_hash_count(self, tmp_path): - """In dry-run mode, stats.hashed counts files that would be hashed.""" - db = tmp_path / "cache.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 30) - - from orcapod.hashing.cache_population import populate_hash_cache - - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db, dry_run=True) - assert stats.hashed == 2 - assert stats.total_bytes_hashed == 50 - - def test_dry_run_already_cached_counted(self, tmp_path): - """dry_run=True counts already-cached files and their bytes correctly.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - stats = populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db, dry_run=True) - - assert stats.already_cached == 1 - assert stats.total_bytes_cached == 20 - assert stats.hashed == 0 - - def test_dry_run_force_all_would_hash(self, tmp_path): - """dry_run=True, force=True: all qualifying files are would_hash regardless of cache.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - stats = populate_hash_cache( - tmp_path, min_size_bytes=_MIN, db_path=db, dry_run=True, force=True - ) - - assert stats.hashed == 1 - assert stats.already_cached == 0 -``` - -- [ ] **Step 4.2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestDryRun tests/test_hashing/test_cache_population.py::TestVisitors -q 2>&1 | tail -10 -``` - -Expected: `ImportError` or `TypeError` — `_DryRunVisitor` and `dry_run` don't exist yet. - -- [ ] **Step 4.3: Add `_DryRunVisitor` to `cache_population.py`** - -After `_HashVisitor`, add: - -```python -class _DryRunVisitor: - """Per-file visitor that checks the cache without hashing or writing. - - Callable with signature ``(resolved, file_stat) -> (FileOutcome, int)``. - Runs on the traversal thread — no thread pool needed for cache lookups. - - Args: - cacher: Shared ``SqliteHashCacher`` instance. - force: If ``True``, skip the cache check and always return ``"would_hash"``. - """ - - def __init__( - self, - cacher: SqliteHashCacher, - *, - force: bool = False, - ) -> None: - self._cacher = cacher - self._force = force - - def __call__( - self, resolved: UPath, file_stat: os.stat_result - ) -> tuple[FileOutcome, int]: - try: - if self._force: - return ("would_hash", file_stat.st_size) - key = FileHashKey(resolved, file_stat.st_mtime_ns, file_stat.st_size) - if self._cacher.get(key) is not None: - return ("cached", file_stat.st_size) - return ("would_hash", file_stat.st_size) - except Exception: - logger.warning("Failed to check file %s", resolved, exc_info=True) - return ("error", 0) -``` - -- [ ] **Step 4.4: Add `dry_run` parameter and serial path to `populate_hash_cache`** - -Add `dry_run: bool = False` to the signature: - -```python -def populate_hash_cache( - path: PathLike | UPath, - *, - min_size_bytes: int = _DEFAULT_MIN_SIZE_BYTES, - db_path: UPath | PathLike | None = None, - algorithm: str = "sha256", - buffer_size: int = 65536, - max_workers: int = _DEFAULT_MAX_WORKERS, - force: bool = False, - dry_run: bool = False, -) -> CachePopulationStats: -``` - -Add to the docstring: - -``` - dry_run: If ``True``, perform the full walk, stat, size filter, and cache check - but skip hashing and cache writes. ``stats.hashed`` reports how many files - *would* be hashed. Defaults to ``False``. - force: If ``True``, re-hash files even if they already have a cache entry. - Defaults to ``False``. -``` - -After the `_excluded` block and before the visitor/accumulator construction, replace the single `visitor = _HashVisitor(...)` line with a branch: - -```python - if dry_run: - visitor: _HashVisitor | _DryRunVisitor = _DryRunVisitor(cacher, force=force) - else: - visitor = _HashVisitor(cacher, hasher, force=force) - accumulator = _Accumulator() -``` - -After the `accumulator = _Accumulator()` line, replace the entire `with ThreadPoolExecutor` block with a branch: - -```python - if dry_run: - # Serial path — cache lookups are fast; no thread pool needed. - stack: list[UPath] = [root] - while stack: - current = stack.pop() - try: - entries = list(current.iterdir()) - except OSError: - logger.warning( - "Cannot access directory %s", current, exc_info=True - ) - accumulator.record_directory_error() - continue - - for entry in entries: - if entry.is_symlink(): - continue - if entry.is_dir(): - stack.append(entry) - continue - if not entry.is_file(): - continue - - try: - resolved = entry.resolve() - if resolved in _excluded: - continue - file_stat = resolved.stat() - except OSError: - logger.warning( - "Cannot stat file %s", entry, exc_info=True - ) - accumulator.record(entry, "error", 0) - continue - - if file_stat.st_size < min_size_bytes: - accumulator.record_skipped_small() - continue - - outcome, nbytes = visitor(resolved, file_stat) - accumulator.record(resolved, outcome, nbytes) - else: - _max_pending = max_workers * _PENDING_FACTOR - pending: dict[Future[tuple[FileOutcome, int]], UPath] = {} - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - stack = [root] - while stack: - current = stack.pop() - try: - entries = list(current.iterdir()) - except OSError: - logger.warning( - "Cannot access directory %s", current, exc_info=True - ) - accumulator.record_directory_error() - continue - - for entry in entries: - if entry.is_symlink(): - continue - if entry.is_dir(): - stack.append(entry) - continue - if not entry.is_file(): - continue - - try: - resolved = entry.resolve() - if resolved in _excluded: - continue - file_stat = resolved.stat() - except OSError: - logger.warning( - "Cannot stat file %s", entry, exc_info=True - ) - accumulator.record(entry, "error", 0) - continue - - if file_stat.st_size < min_size_bytes: - accumulator.record_skipped_small() - continue - - if len(pending) >= _max_pending: - done_futures, _ = wait(pending.keys(), return_when=FIRST_COMPLETED) - for fut in done_futures: - p = pending.pop(fut) - outcome, nbytes = fut.result() - accumulator.record(p, outcome, nbytes) - - future = executor.submit(visitor, resolved, file_stat) - pending[future] = resolved - - for fut in as_completed(pending): - p = pending[fut] - outcome, nbytes = fut.result() - accumulator.record(p, outcome, nbytes) - - return accumulator.finalize() -``` - -The `return accumulator.finalize()` line sits at the same indentation level as the -`with SqliteHashCacher(...)` block — i.e., it is *inside* `populate_hash_cache` but -*outside* the cacher context manager. `accumulator` is still in scope because Python -local variables outlive the `with` block that created them. - -- [ ] **Step 4.5: Run all tests** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -10 -``` - -Expected: `35 passed`. - -- [ ] **Step 4.6: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): add dry_run parameter and _DryRunVisitor (ITL-500)" -``` - ---- - -## Task 5: Add `progress_callback` parameter - -**Files:** -- Modify: `src/orcapod/hashing/cache_population.py` -- Modify: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 5.1: Write failing tests** - -Add to `tests/test_hashing/test_cache_population.py`: - -```python -class TestProgressCallback: - def test_callback_fires_once_per_qualifying_file(self, tmp_path): - """Callback is called exactly once for each file that passes the size filter.""" - db = tmp_path / "cache.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - calls = [] - populate_hash_cache( - tmp_path, - min_size_bytes=_MIN, - db_path=db, - progress_callback=lambda path, outcome, stats: calls.append(outcome), - ) - - assert len(calls) == 2 - assert all(o == "hashed" for o in calls) - - def test_callback_receives_correct_path(self, tmp_path): - """Callback path argument is the resolved path of the file just processed.""" - db = tmp_path / "cache.db" - f = _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - paths_seen = [] - populate_hash_cache( - tmp_path, - min_size_bytes=_MIN, - db_path=db, - progress_callback=lambda path, outcome, stats: paths_seen.append(path), - ) - - assert len(paths_seen) == 1 - assert paths_seen[0] == f.resolve() - - def test_callback_not_fired_for_skipped_small(self, tmp_path): - """Files below min_size_bytes do not trigger the callback.""" - db = tmp_path / "cache.db" - _write(tmp_path, "small.bin", 5) # below _MIN=10 - _write(tmp_path, "big.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - calls = [] - populate_hash_cache( - tmp_path, - min_size_bytes=_MIN, - db_path=db, - progress_callback=lambda path, outcome, stats: calls.append(outcome), - ) - - assert len(calls) == 1 - assert calls[0] == "hashed" - - def test_callback_running_totals_are_accurate(self, tmp_path): - """Each callback invocation receives a snapshot with totals correct up to that point.""" - db = tmp_path / "cache.db" - _write(tmp_path, "a.bin", 20) - _write(tmp_path, "b.bin", 30) - - from orcapod.hashing.cache_population import populate_hash_cache - - snapshots = [] - populate_hash_cache( - tmp_path, - min_size_bytes=_MIN, - db_path=db, - progress_callback=lambda path, outcome, stats: snapshots.append(stats), - ) - - assert len(snapshots) == 2 - # After both files are processed the last snapshot has hashed=2 and correct total bytes. - last = snapshots[-1] - assert last.hashed == 2 - assert last.total_bytes_hashed == 50 - - def test_callback_outcome_cached(self, tmp_path): - """On second run, callback receives 'cached' outcome.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - populate_hash_cache(tmp_path, min_size_bytes=_MIN, db_path=db) - - outcomes = [] - populate_hash_cache( - tmp_path, - min_size_bytes=_MIN, - db_path=db, - progress_callback=lambda path, outcome, stats: outcomes.append(outcome), - ) - - assert outcomes == ["cached"] - - def test_callback_outcome_would_hash_in_dry_run(self, tmp_path): - """In dry-run mode, callback receives 'would_hash' for uncached files.""" - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - from orcapod.hashing.cache_population import populate_hash_cache - - outcomes = [] - populate_hash_cache( - tmp_path, - min_size_bytes=_MIN, - db_path=db, - dry_run=True, - progress_callback=lambda path, outcome, stats: outcomes.append(outcome), - ) - - assert outcomes == ["would_hash"] -``` - -- [ ] **Step 5.2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestProgressCallback -q 2>&1 | tail -10 -``` - -Expected: `TypeError: populate_hash_cache() got an unexpected keyword argument 'progress_callback'` - -- [ ] **Step 5.3: Add `ProgressCallback` type alias to `cache_population.py`** - -After the `FileOutcome` line, add: - -```python -ProgressCallback = Callable[["Path", FileOutcome, CachePopulationStats], None] -``` - -Also ensure `Callable` and `Path` are imported. Replace the forward reference in `_Accumulator.__init__` with the real type now that `ProgressCallback` is defined before `_Accumulator`. - -At the top of the file, update the imports: - -```python -from pathlib import Path -from typing import Callable, Literal -``` - -Update `_Accumulator.__init__` signature to use the real type: - -```python - def __init__(self, callback: ProgressCallback | None = None) -> None: -``` - -- [ ] **Step 5.4: Add `progress_callback` parameter to `populate_hash_cache`** - -Add to the signature: - -```python -def populate_hash_cache( - path: PathLike | UPath, - *, - min_size_bytes: int = _DEFAULT_MIN_SIZE_BYTES, - db_path: UPath | PathLike | None = None, - algorithm: str = "sha256", - buffer_size: int = 65536, - max_workers: int = _DEFAULT_MAX_WORKERS, - force: bool = False, - dry_run: bool = False, - progress_callback: ProgressCallback | None = None, -) -> CachePopulationStats: -``` - -Add to the docstring: - -``` - progress_callback: Optional callable invoked once per qualifying file - (i.e. files that pass the size filter). Receives the resolved - ``Path``, a ``FileOutcome`` string, and a frozen - ``CachePopulationStats`` snapshot of running totals at that moment. - Not called for files below ``min_size_bytes`` or for directory - access errors. Defaults to ``None``. -``` - -Pass the callback to `_Accumulator`: - -```python - accumulator = _Accumulator(progress_callback) -``` - -- [ ] **Step 5.5: Run all tests** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -10 -``` - -Expected: `41 passed`. - -- [ ] **Step 5.6: Commit** - -```bash -git add src/orcapod/hashing/cache_population.py tests/test_hashing/test_cache_population.py -git commit -m "feat(hashing): add progress_callback parameter to populate_hash_cache (ITL-500)" -``` - ---- - -## Task 6: Update CLI with `--dry-run`, `--force`, and cached-bytes output - -**Files:** -- Modify: `src/orcapod/cli/warm_cache.py` -- Modify: `tests/test_hashing/test_cache_population.py` - -- [ ] **Step 6.1: Write failing CLI tests** - -Add to `tests/test_hashing/test_cache_population.py`. First, add the import at the top of the file: - -```python -from typer.testing import CliRunner -``` - -Then add the test class: - -```python -class TestCLI: - def _app(self): - import typer - from orcapod.cli.warm_cache import warm_cache - app = typer.Typer() - app.command()(warm_cache) - return app - - def test_force_flag_wires_through(self, tmp_path): - """--force causes already-cached files to be re-hashed.""" - runner = CliRunner() - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - app = self._app() - # First run to populate cache. - runner.invoke(app, [str(tmp_path), "--min-size", "0", "--db-path", str(db)]) - # Second run with --force. - result = runner.invoke( - app, [str(tmp_path), "--min-size", "0", "--db-path", str(db), "--force"] - ) - assert result.exit_code == 0 - assert "1 hashed" in result.output - assert "0 already cached" in result.output - - def test_dry_run_flag_wires_through(self, tmp_path): - """--dry-run prints 'would be hashed' and makes no cache writes.""" - runner = CliRunner() - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - app = self._app() - result = runner.invoke( - app, [str(tmp_path), "--min-size", "0", "--db-path", str(db), "--dry-run"] - ) - assert result.exit_code == 0 - assert "would be hashed" in result.output - assert "Dry run" in result.output - - # Cache must be empty after dry-run. - real_run = runner.invoke( - app, [str(tmp_path), "--min-size", "0", "--db-path", str(db)] - ) - assert "1 hashed" in real_run.output - - def test_normal_output_includes_cached_gb(self, tmp_path): - """Normal run output shows cached GB on the second run.""" - runner = CliRunner() - db = tmp_path / "cache.db" - _write(tmp_path, "f.bin", 20) - - app = self._app() - runner.invoke(app, [str(tmp_path), "--min-size", "0", "--db-path", str(db)]) - result = runner.invoke( - app, [str(tmp_path), "--min-size", "0", "--db-path", str(db)] - ) - assert result.exit_code == 0 - # "1 already cached (0.00 GB)" should appear. - assert "already cached" in result.output - assert "GB" in result.output -``` - -- [ ] **Step 6.2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py::TestCLI -q 2>&1 | tail -10 -``` - -Expected: failures — `--force` and `--dry-run` flags don't exist yet. - -- [ ] **Step 6.3: Update `warm_cache.py`** - -Replace the entire contents of `src/orcapod/cli/warm_cache.py` with: - -```python -"""``orcapod warm-cache`` subcommand. - -Pre-populates the SQLite file-hash cache for large files under a target -directory so that subsequent pipeline runs skip expensive content hashing. -""" - -from __future__ import annotations - -from pathlib import Path - -import typer -from upath import UPath - -_DEFAULT_MIN_SIZE_MB: float = 500.0 - - -def warm_cache( - path: str = typer.Argument(..., help="Root directory to scan recursively."), - min_size: float = typer.Option( - _DEFAULT_MIN_SIZE_MB, - "--min-size", - help="Minimum file size in MB. Files smaller than this are skipped. Default: 500 MB.", - show_default=True, - ), - db_path: str | None = typer.Option( - None, - "--db-path", - help=( - "Path to the SQLite hash-cache database. " - "Defaults to $ORCAPOD_HASH_CACHE_DB or ~/.orcapod/file_hash_cache.db." - ), - ), - algorithm: str = typer.Option( - "sha256", - "--algorithm", - help="Hash algorithm (sha256, xxh64, md5, …). Default: sha256.", - show_default=True, - ), - buffer_size: int = typer.Option( - 65536, - "--buffer-size", - help="Read buffer size in bytes. Default: 65536.", - show_default=True, - ), - max_workers: int = typer.Option( - 4, - "--workers", - help="Number of threads for concurrent hashing. Default: 4.", - show_default=True, - ), - dry_run: bool = typer.Option( - False, - "--dry-run", - help="Scan and check cache without hashing. Prints what would be done.", - ), - force: bool = typer.Option( - False, - "--force", - help="Re-hash files even if already cached.", - ), -) -> None: - """Pre-populate the file-hash cache for large files under PATH. - - Recursively scans PATH and hashes every file that is at least MIN_SIZE MB. - Files already present in the cache are skipped unless --force is given. - Use --dry-run to preview what would be done without writing to the cache. - On completion, prints a summary with counts and throughput. - """ - from orcapod.hashing.cache_population import populate_hash_cache - - min_size_bytes = int(min_size * 1024 * 1024) - _db_path: Path | None = Path(db_path) if db_path is not None else None - - if max_workers < 1: - typer.echo("Error: --workers must be at least 1", err=True) - raise typer.Exit(code=1) - - root = UPath(path) - if not root.exists(): - typer.echo(f"Error: path does not exist: {path}", err=True) - raise typer.Exit(code=1) - if not root.is_dir(): - typer.echo(f"Error: path is not a directory: {path}", err=True) - raise typer.Exit(code=1) - - if dry_run: - typer.echo(f"Dry-run scan of {path} ...") - else: - typer.echo(f"Scanning {path} ...") - - stats = populate_hash_cache( - path, - min_size_bytes=min_size_bytes, - db_path=_db_path, - algorithm=algorithm, - buffer_size=buffer_size, - max_workers=max_workers, - dry_run=dry_run, - force=force, - ) - - gb_hashed = stats.total_bytes_hashed / (1024**3) - gb_cached = stats.total_bytes_cached / (1024**3) - speed_gb = stats.avg_hashing_speed / (1024**3) - min_size_display = f"{min_size:g} MB" - - if dry_run: - typer.echo( - f"Dry run complete in {stats.total_duration:.1f}s — " - f"{stats.hashed} would be hashed ({gb_hashed:.2f} GB), " - f"{stats.already_cached} already cached ({gb_cached:.2f} GB), " - f"{stats.skipped_small} skipped (< {min_size_display}), " - f"{stats.errors} errors." - ) - else: - typer.echo( - f"Done in {stats.total_duration:.1f}s — " - f"{stats.hashed} hashed ({gb_hashed:.2f} GB), " - f"{stats.already_cached} already cached ({gb_cached:.2f} GB), " - f"{stats.skipped_small} skipped (< {min_size_display}), " - f"{stats.errors} errors." - ) - if stats.hashed > 0: - typer.echo(f"Average hashing speed: {speed_gb:.2f} GB/s") -``` - -- [ ] **Step 6.4: Run all tests** - -```bash -uv run pytest tests/test_hashing/test_cache_population.py -q 2>&1 | tail -10 -``` - -Expected: `44 passed`. - -- [ ] **Step 6.5: Run the full test suite to check for regressions** - -```bash -uv run pytest -q 2>&1 | tail -15 -``` - -Expected: all tests pass. - -- [ ] **Step 6.6: Commit** - -```bash -git add src/orcapod/cli/warm_cache.py tests/test_hashing/test_cache_population.py -git commit -m "feat(cli): add --dry-run and --force to orcapod warm-cache (ITL-500)" -``` - ---- - -## Task 7: Expose new types in public API and push branch - -**Files:** -- Modify: `src/orcapod/hashing/__init__.py` - -- [ ] **Step 7.1: Export `FileOutcome` and `ProgressCallback` from `orcapod.hashing`** - -In `src/orcapod/hashing/__init__.py`, update the import from `cache_population`: - -```python -from orcapod.hashing.cache_population import ( - CachePopulationStats, - FileOutcome, - ProgressCallback, - populate_hash_cache, -) -``` - -Add `"FileOutcome"` and `"ProgressCallback"` to `__all__`: - -```python - "CachePopulationStats", - "FileOutcome", - "ProgressCallback", - "populate_hash_cache", -``` - -- [ ] **Step 7.2: Verify the new names are importable** - -```bash -uv run python -c "from orcapod.hashing import FileOutcome, ProgressCallback, CachePopulationStats, populate_hash_cache; print('OK')" -``` - -Expected: `OK` - -- [ ] **Step 7.3: Run the full test suite one final time** - -```bash -uv run pytest -q 2>&1 | tail -10 -``` - -Expected: all tests pass. - -- [ ] **Step 7.4: Commit** - -```bash -git add src/orcapod/hashing/__init__.py -git commit -m "feat(hashing): export FileOutcome and ProgressCallback from orcapod.hashing (ITL-500)" -``` - -- [ ] **Step 7.5: Push the feature branch** - -```bash -git push -u origin eywalker/itl-500-improve-populate_hash_cache-ergonomics-progress-callback-dry -``` diff --git a/superpowers/plans/2026-07-07-dict-source-auto-registration.md b/superpowers/plans/2026-07-07-dict-source-auto-registration.md deleted file mode 100644 index d777b617a..000000000 --- a/superpowers/plans/2026-07-07-dict-source-auto-registration.md +++ /dev/null @@ -1,226 +0,0 @@ -# DictSource Auto-Registration Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `DictSource` automatically register Pydantic `BaseModel` and `@dataclass` column types so users never need to pre-register them via a function pod. - -**Architecture:** Insert one call to `ensure_types_registered_for_schemas(python_schema)` in `UniversalTypeConverter.python_dicts_to_arrow_table` immediately before the Python schema is converted to an Arrow schema. This mirrors the existing function-pod eager-registration pattern and is idempotent, thread-safe, and a no-op for already-registered or primitive types. - -**Tech Stack:** Python, PyArrow, Pydantic v2, `dataclasses` stdlib, pytest (run via `uv run pytest`) - ---- - -## File Map - -| Action | Path | What changes | -|--------|------|--------------| -| Modify | `src/orcapod/semantic_types/universal_converter.py` | Add 1 line in `python_dicts_to_arrow_table` | -| Create | `tests/test_core/sources/test_dict_source_auto_registration.py` | 4 new tests | - ---- - -## Task 1: Write the Failing Tests - -**Files:** -- Create: `tests/test_core/sources/test_dict_source_auto_registration.py` - -- [ ] **Step 1.1 — Write the test file** - -```python -"""Tests for DictSource auto-registration of Pydantic / dataclass column types. - -Verifies that DictSource construction and iteration succeed for Pydantic BaseModel -and @dataclass column types that have never been registered via a function pod. -""" -from __future__ import annotations - -import dataclasses - -import pytest -from pydantic import BaseModel - -from orcapod.core.sources import DictSource - - -# --------------------------------------------------------------------------- -# Fixtures — fresh model classes scoped per-test to avoid cross-test registry -# state. Using classes defined at module level is fine because registration is -# idempotent; the classes just need to be new relative to any prior test run. -# --------------------------------------------------------------------------- - - -class _Point(BaseModel): - x: float - y: float - - -@dataclasses.dataclass -class _Measurement: - value: float - unit: str - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestDictSourceAutoRegistration: - def test_pydantic_model_no_prior_registration(self): - """DictSource with a fresh Pydantic model column succeeds end-to-end.""" - src = DictSource( - data=[ - {"id": 1, "pt": _Point(x=1.0, y=2.0)}, - {"id": 2, "pt": _Point(x=3.0, y=4.0)}, - ], - tag_columns=["id"], - ) - rows = list(src.iter_data()) - assert len(rows) == 2 - - def test_dataclass_no_prior_registration(self): - """DictSource with a fresh @dataclass column succeeds end-to-end.""" - src = DictSource( - data=[ - {"id": 1, "m": _Measurement(value=1.5, unit="mm")}, - {"id": 2, "m": _Measurement(value=2.5, unit="cm")}, - ], - tag_columns=["id"], - ) - rows = list(src.iter_data()) - assert len(rows) == 2 - - def test_pydantic_no_double_registration_error(self): - """Creating a second DictSource with the same Pydantic model type does not raise.""" - data = [{"id": 1, "pt": _Point(x=0.0, y=0.0)}] - DictSource(data=data, tag_columns=["id"]) - # Second construction — type already in registry; must be idempotent. - src2 = DictSource(data=data, tag_columns=["id"]) - rows = list(src2.iter_data()) - assert len(rows) == 1 - - def test_pod_registration_not_broken(self): - """Function-pod eager registration still works after DictSource auto-registration.""" - from orcapod.core.data_function import PythonDataFunction - from orcapod.core.function_pod import FunctionPod - - class _Sensor(BaseModel): - reading: float - - def identity(sensor: _Sensor) -> _Sensor: - return sensor - - # Eager registration via pod construction — must not raise. - pod = FunctionPod( - data_function=PythonDataFunction(identity, output_keys="sensor") - ) - assert pod is not None - - # DictSource with the same type — registration already done; must not raise. - src = DictSource( - data=[{"id": 1, "sensor": _Sensor(reading=42.0)}], - tag_columns=["id"], - ) - rows = list(src.iter_data()) - assert len(rows) == 1 -``` - -- [ ] **Step 1.2 — Run the tests to confirm they fail** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_core/sources/test_dict_source_auto_registration.py -v -``` - -Expected: All four tests **FAIL** with `ValueError: Unsupported Python type: ...` (for the first three) and the pod test may pass or fail depending on whether the fix is in. All failing with that error confirms the tests exercise the right path. - ---- - -## Task 2: Implement the Fix - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` (around line 956–959) - -- [ ] **Step 2.1 — Apply the one-line change** - -Open `src/orcapod/semantic_types/universal_converter.py`. Find `python_dicts_to_arrow_table`. The current `if arrow_schema is None:` block (starting around line 956) reads: - -```python - if arrow_schema is None: - # Convert to Arrow schema - assert python_schema is not None, "Python schema should not be None here" - arrow_schema = self.python_schema_to_arrow_schema(python_schema) -``` - -Replace it with: - -```python - if arrow_schema is None: - # Convert to Arrow schema — auto-register any Pydantic / dataclass types - # encountered here so DictSource users do not need to pre-register via a - # function pod. ensure_types_registered_for_schemas is idempotent and - # thread-safe; it is a no-op for primitives and already-registered types. - assert python_schema is not None, "Python schema should not be None here" - self.ensure_types_registered_for_schemas(python_schema) - arrow_schema = self.python_schema_to_arrow_schema(python_schema) -``` - -- [ ] **Step 2.2 — Run the new tests to confirm they pass** - -```bash -uv run pytest tests/test_core/sources/test_dict_source_auto_registration.py -v -``` - -Expected output (all green): - -``` -PASSED tests/test_core/sources/test_dict_source_auto_registration.py::TestDictSourceAutoRegistration::test_pydantic_model_no_prior_registration -PASSED tests/test_core/sources/test_dict_source_auto_registration.py::TestDictSourceAutoRegistration::test_dataclass_no_prior_registration -PASSED tests/test_core/sources/test_dict_source_auto_registration.py::TestDictSourceAutoRegistration::test_pydantic_no_double_registration_error -PASSED tests/test_core/sources/test_dict_source_auto_registration.py::TestDictSourceAutoRegistration::test_pod_registration_not_broken -``` - -- [ ] **Step 2.3 — Run the full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests pass (exit code 0). If any pre-existing failures appear, check that they existed on `main` before this branch — do not count them as regressions introduced by this change. - -- [ ] **Step 2.4 — Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py \ - tests/test_core/sources/test_dict_source_auto_registration.py -git commit -m "fix(universal_converter): auto-register Pydantic/dataclass types in python_dicts_to_arrow_table (ITL-446)" -``` - ---- - -## Task 3: Commit the Plan - -**Files:** -- Commit: `superpowers/plans/2026-07-07-dict-source-auto-registration.md` (this file) - -- [ ] **Step 3.1 — Commit the plan doc** - -```bash -git add superpowers/plans/2026-07-07-dict-source-auto-registration.md -git commit -m "docs(plans): add implementation plan for DictSource auto-registration (ITL-446)" -``` - ---- - -## Self-Review Checklist - -**Spec coverage:** -- [x] `python_dicts_to_arrow_table` patched — Task 2. -- [x] Pydantic test, no prior registration — Task 1, `test_pydantic_model_no_prior_registration`. -- [x] Dataclass test, no prior registration — Task 1, `test_dataclass_no_prior_registration`. -- [x] Idempotency (no double-registration error) — Task 1, `test_pydantic_no_double_registration_error`. -- [x] No pod regression — Task 1, `test_pod_registration_not_broken`. - -**Placeholder scan:** No TBDs, no vague "add error handling" steps. ✓ - -**Type consistency:** `ensure_types_registered_for_schemas` used in Task 2 is the same method called in Task 1's test (indirectly via DictSource). ✓ diff --git a/superpowers/plans/2026-07-07-itl-142-pydantic-missing-tests.md b/superpowers/plans/2026-07-07-itl-142-pydantic-missing-tests.md deleted file mode 100644 index 56498027d..000000000 --- a/superpowers/plans/2026-07-07-itl-142-pydantic-missing-tests.md +++ /dev/null @@ -1,205 +0,0 @@ -# ITL-142: Pydantic Missing Tests Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add two missing tests to `test_pydantic_logical_type_factory.py` covering `@computed_field` exclusion and validator behaviour on decode — the only genuine gaps between the current implementation and the ITL-142 success criteria. - -**Architecture:** Both behaviours are already correct in `PydanticLogicalTypeFactory`; only test coverage is absent. The implementation walks `model_fields` (not `model_computed_fields`), so computed fields are naturally excluded. Reconstruction calls `Model(**kwargs)`, so Pydantic validators run automatically. We verify both by writing tests against the existing factory code. - -**Tech Stack:** Python 3.12, Pydantic v2, PyArrow, pytest. Run tests with `uv run pytest`. - ---- - -### Task 1: Add test for `@computed_field` exclusion - -**Files:** -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` (append after line 646) - -- [ ] **Step 1: Add the module-level model with a computed field** - - First, update the existing pydantic import at the top of the file (line 11) to add `computed_field`: - - ```python - from pydantic import BaseModel, PrivateAttr, computed_field - ``` - - Then append the following model definition to the module-level model block (after `_LiteralRoundTripModel`, around line 201). It must be at module scope so its FQCN is importable. - - ```python - class _ModelWithComputedField(BaseModel): - x: int - y: int - - @computed_field - @property - def total(self) -> int: - return self.x + self.y - ``` - -- [ ] **Step 2: Write the failing test** - - Append after the last test in the file: - - ```python - def test_computed_field_not_stored(): - """@computed_field properties must not appear in the Arrow struct schema. - - PydanticLogicalTypeFactory walks model_fields (not model_computed_fields), - so @computed_field properties are naturally excluded. This test pins that - behaviour so a future refactor cannot accidentally include them. - """ - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - factory = PydanticLogicalTypeFactory() - converter = _make_full_converter() - lt = factory.create_for_python_type(_ModelWithComputedField, converter=converter) - - storage = lt.get_arrow_extension_type().storage_type - field_names = {storage.field(i).name for i in range(storage.num_fields)} - assert "x" in field_names - assert "y" in field_names - assert "total" not in field_names, ( - "@computed_field 'total' must not appear in the Arrow struct schema" - ) - assert storage.num_fields == 2 - ``` - -- [ ] **Step 3: Run the test to verify it fails** (should PASS — but confirm it exists and runs) - - ```bash - uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py::test_computed_field_not_stored -v - ``` - - Expected: `PASSED` — this confirms the behaviour is correct and the test is wired up. - ---- - -### Task 2: Add test for validator behaviour on decode - -**Files:** -- Modify: `tests/test_extension_types/test_pydantic_logical_type_factory.py` - -- [ ] **Step 1: Add the module-level model with a validator** - - Update the existing pydantic import (already updated in Task 1) to also add `field_validator`: - - ```python - from pydantic import BaseModel, PrivateAttr, computed_field, field_validator - ``` - - Then append this model after `_ModelWithComputedField`: - - ```python - class _ModelWithValidator(BaseModel): - name: str - - @field_validator("name") - @classmethod - def name_must_be_uppercase(cls, v: str) -> str: - return v.upper() - ``` - -- [ ] **Step 2: Write the failing test** - - Append after `test_computed_field_not_stored`: - - ```python - def test_validator_runs_on_decode(): - """Pydantic validators must execute when a model is reconstructed from Arrow storage. - - Reconstruction calls Model(**kwargs), which runs Pydantic's validation pipeline - automatically. This test uses a @field_validator that uppercases the 'name' field - to provide a concrete, observable signal that validators are running. - - Note: non-idempotent validators (those that mutate values, like this one) mean - the decoded instance may differ from a round-trip re-encoding of the original - value — this is a Pydantic property, not an Orcapod bug. - """ - from orcapod.extension_types.pydantic_logical_type_factory import PydanticLogicalTypeFactory - - converter = _make_full_converter() - factory = PydanticLogicalTypeFactory() - lt = factory.create_for_python_type(_ModelWithValidator, converter=converter) - converter.register_logical_type(lt) - - # Store a lowercase value directly (bypassing validator on write for clarity) - storage_value = {"name": "alice"} - - # Decode — validator must run and uppercase the name - reconstructed = lt.storage_to_python(storage_value, converter) - assert isinstance(reconstructed, _ModelWithValidator) - assert reconstructed.name == "ALICE", ( - "Validator did not run on decode — expected 'alice' to be uppercased to 'ALICE'" - ) - ``` - -- [ ] **Step 3: Run the new test** - - ```bash - uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py::test_validator_runs_on_decode -v - ``` - - Expected: `PASSED`. - ---- - -### Task 3: Run the full test file and commit - -- [ ] **Step 1: Run the full pydantic test file** - - ```bash - uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v - ``` - - Expected: `33 passed` (31 existing + 2 new). - -- [ ] **Step 2: Commit** - - ```bash - git add tests/test_extension_types/test_pydantic_logical_type_factory.py - git commit -m "test(extension_types): add computed_field exclusion and validator-on-decode tests for PydanticLogicalTypeFactory (ITL-142)" - ``` - ---- - -### Task 4: Create PR - -- [ ] **Step 1: Push branch** - - ```bash - git push -u origin eywalker/itl-142-add-pydantic-v2-support-to-orcapod-via-arrow-struct-encoding - ``` - -- [ ] **Step 2: Create PR against main** - - ```bash - gh pr create \ - --title "test(extension_types): add computed_field and validator tests for pydantic (ITL-142)" \ - --base main \ - --body "$(cat <<'EOF' - ## Summary - - Closes ITL-142. - - Pydantic v2 `BaseModel` support was already fully implemented and automatically - registered in the default context (`v0.1.json`) — this PR fills the last two test - gaps identified during the ITL-142 verification pass: - - - **`test_computed_field_not_stored`**: pins that `@computed_field` properties are - excluded from the Arrow struct schema (only `model_fields` is walked, not - `model_computed_fields`). - - **`test_validator_runs_on_decode`**: confirms that Pydantic validators run during - reconstruction from Arrow storage (via `Model(**kwargs)`), and documents that - non-idempotent validators may produce asymmetric round-trips. - - No production code changes — both behaviours were already correct. - - ## Test plan - - - [ ] `uv run pytest tests/test_extension_types/test_pydantic_logical_type_factory.py -v` → 33 passed - - [ ] `uv run pytest tests/test_extension_types/ tests/test_hashing/test_pydantic_dataclass_hashing.py -v` → all passed - - 🤖 Generated with [Claude Code](https://claude.com/claude-code) - EOF - )" - ``` diff --git a/superpowers/plans/2026-07-07-itl-507-ephemeral-function-pod-output.md b/superpowers/plans/2026-07-07-itl-507-ephemeral-function-pod-output.md deleted file mode 100644 index eaea73ecc..000000000 --- a/superpowers/plans/2026-07-07-itl-507-ephemeral-function-pod-output.md +++ /dev/null @@ -1,1788 +0,0 @@ -# Ephemeral Function-Pod Output v1 Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `NodeConfig.ephemeral_result=True` mode to `FunctionJobNode` so new results route to an `InMemoryArrowDatabase` instead of the persistent result DB, while persistent cache hits remain available. - -**Architecture:** Add an `IS_EPHEMERAL` boolean column to the pipeline tag table (replacing the "temp:" string prefix described in the spec, since `DATA_RECORD_ID` is `pa.large_binary()` and cannot carry a string prefix). `FunctionJobNode` gains an `_ephemeral_cached_pod` slot (a `CachedFunctionPod` backed by `InMemoryArrowDatabase`) created in `set_ephemeral_store()`. `_fetch_joined_records` is split into two independent Polars joins (one per store) merged with an anti-join that gives persistent results priority. `AbstractPipelineBase.set_ephemeral_store` propagates the store to every node. - -**Tech Stack:** PyArrow, Polars (already used in `_fetch_joined_records`), existing `CachedFunctionPod`, `InMemoryArrowDatabase`, `ResultCache`. - ---- - -## File Map - -| File | Change | -|---|---| -| `src/orcapod/system_constants.py` | Add `IS_EPHEMERAL_COL` property to `SystemConstant` | -| `src/orcapod/types.py` | Add `ephemeral_result: bool = False` to `NodeConfig` | -| `src/orcapod/protocols/node_protocols.py` | Add `set_ephemeral_store()` to `FunctionNodeProtocol` and `OperatorNodeProtocol` | -| `src/orcapod/protocols/pipeline_protocols.py` | Add `set_ephemeral_store()` to `PipelineProtocol` | -| `src/orcapod/core/nodes/function_node.py` | No-op on `FunctionNodeBase`; override + two-store logic on `FunctionJobNode` | -| `src/orcapod/core/nodes/operator_node.py` | No-op `set_ephemeral_store()` on `OperatorNodeBase` | -| `src/orcapod/pipeline/base.py` | `AbstractPipelineBase.set_ephemeral_store()` propagating to all nodes | -| `tests/test_core/function_pod/test_ephemeral_result.py` | New file — all 17 test scenarios | - ---- - -## Task 1: Add `IS_EPHEMERAL_COL` constant - -**Files:** -- Modify: `src/orcapod/system_constants.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_system_constants.py (append to existing file, or create if absent) -def test_is_ephemeral_col(): - from orcapod.system_constants import constants - col = constants.IS_EPHEMERAL_COL - assert col.startswith("__"), f"expected META_PREFIX, got {col!r}" - assert "is_ephemeral" in col -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/ -k "test_is_ephemeral_col" -v -``` -Expected: FAIL with `AttributeError: 'SystemConstant' object has no attribute 'IS_EPHEMERAL_COL'` - -- [ ] **Step 3: Add the constant** - -In `src/orcapod/system_constants.py`, add at the module level (after the existing constants block, around line 17): - -```python -IS_EPHEMERAL = "is_ephemeral" -``` - -Add the property to `SystemConstant` (after the `ENV_INFO` property, around line 101): - -```python -@property -def IS_EPHEMERAL_COL(self) -> str: - return f"{self._global_prefix}{SYSTEM_COLUMN_PREFIX}{IS_EPHEMERAL}" -``` - -- [ ] **Step 4: Run test to verify it passes** - -``` -uv run pytest tests/ -k "test_is_ephemeral_col" -v -``` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/system_constants.py tests/ -git commit -m "feat(system_constants): add IS_EPHEMERAL_COL for ephemeral pipeline record routing" -``` - ---- - -## Task 2: `NodeConfig.ephemeral_result` field - -**Files:** -- Modify: `src/orcapod/types.py:334-345` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_core/test_node_config.py (create if absent, or append) -def test_node_config_ephemeral_result_default_false(): - from orcapod.types import NodeConfig - cfg = NodeConfig() - assert cfg.ephemeral_result is False - - -def test_node_config_ephemeral_result_true(): - from orcapod.types import NodeConfig - cfg = NodeConfig(ephemeral_result=True) - assert cfg.ephemeral_result is True - - -def test_node_config_ephemeral_result_is_frozen(): - from orcapod.types import NodeConfig - import dataclasses - cfg = NodeConfig(ephemeral_result=True) - with pytest.raises((dataclasses.FrozenInstanceError, TypeError)): - cfg.ephemeral_result = False # type: ignore[misc] -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/ -k "test_node_config_ephemeral" -v -``` -Expected: FAIL with `TypeError` (unexpected keyword argument) or `AttributeError`. - -- [ ] **Step 3: Add the field** - -In `src/orcapod/types.py`, edit the `NodeConfig` class (lines 334-345): - -```python -@dataclass(frozen=True, slots=True) -class NodeConfig: - """Per-node execution configuration. - - Attributes: - max_concurrency: Override for this node's concurrency limit. - ``None`` inherits from ``PipelineConfig.default_max_concurrency``. - ``1`` means sequential (rate-limited APIs, preserves ordering). - ephemeral_result: If ``True``, new computation results are written to - the pipeline-scoped ephemeral store (``InMemoryArrowDatabase``) - instead of the persistent result database. Persistent cache hits - are still served when available. Raises ``RuntimeError`` at - execution time if ``ephemeral_result=True`` but no ephemeral - store has been injected via ``set_ephemeral_store()``. - """ - - max_concurrency: int | None = None - ephemeral_result: bool = False -``` - -- [ ] **Step 4: Run test to verify it passes** - -``` -uv run pytest tests/ -k "test_node_config_ephemeral" -v -``` -Expected: PASS - -- [ ] **Step 5: Run the full test suite to catch regressions** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass (no existing tests should break — new field has a default). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/types.py tests/ -git commit -m "feat(types): add ephemeral_result field to NodeConfig" -``` - ---- - -## Task 3: Protocol additions — `set_ephemeral_store` - -**Files:** -- Modify: `src/orcapod/protocols/node_protocols.py` -- Modify: `src/orcapod/protocols/pipeline_protocols.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_protocols/test_ephemeral_store_protocol.py (new file) -import pytest -from orcapod.protocols.node_protocols import FunctionNodeProtocol, OperatorNodeProtocol -from orcapod.protocols.pipeline_protocols import PipelineProtocol - - -def test_function_node_protocol_has_set_ephemeral_store(): - assert hasattr(FunctionNodeProtocol, "set_ephemeral_store") - - -def test_operator_node_protocol_has_set_ephemeral_store(): - assert hasattr(OperatorNodeProtocol, "set_ephemeral_store") - - -def test_pipeline_protocol_has_set_ephemeral_store(): - assert hasattr(PipelineProtocol, "set_ephemeral_store") -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_protocols/ -k "test_ephemeral_store_protocol" -v -``` -Expected: FAIL with `AttributeError` - -- [ ] **Step 3: Add `set_ephemeral_store` to `node_protocols.py`** - -In `src/orcapod/protocols/node_protocols.py`, add the import for `InMemoryArrowDatabase` under `TYPE_CHECKING` (it's already imported indirectly; add explicitly): - -```python -if TYPE_CHECKING: - from orcapod.channels import ReadableChannel, WritableChannel - from orcapod.core.nodes import GraphNode - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol - from orcapod.protocols.core_protocols import ( - DataProtocol, - StreamProtocol, - TagProtocol, - ) -``` - -Add `set_ephemeral_store` to `FunctionNodeProtocol` (after the `async_execute` method, around line 90): - -```python -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """Assign or remove the ephemeral result store for this node. - - Pass an ``InMemoryArrowDatabase`` to attach the store. - Pass ``None`` to detach it — the node falls back to persistent-only - behaviour for subsequent writes. No-op for node types that do not - support ephemeral result storage (e.g. blueprint ``FunctionNode``). - """ - ... -``` - -Add the same method to `OperatorNodeProtocol` (after `async_execute`, around line 116): - -```python -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """Assign or remove the ephemeral result store for this node. - - No-op for operator nodes in v1 — full ephemeral support for operators - is deferred to ITL-509. - """ - ... -``` - -- [ ] **Step 4: Add `set_ephemeral_store` to `pipeline_protocols.py`** - -In `src/orcapod/protocols/pipeline_protocols.py`, add `InMemoryArrowDatabase` under `TYPE_CHECKING`: - -```python -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase - from orcapod.pipeline.dag import GraphProtocol -``` - -Add the method to `PipelineProtocol` (after the `dag` property, around line 60): - -```python -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """Assign or remove the ephemeral result store for all nodes. - - Propagates ``store`` to every node in the pipeline by calling - ``node.set_ephemeral_store(store)`` on each. Nodes that do not - support ephemeral storage (e.g. operator nodes in v1) treat this - as a no-op. - - Pass ``None`` to detach the ephemeral store from all nodes, - reverting them to persistent-only writes for subsequent runs. - """ - ... -``` - -- [ ] **Step 5: Run test to verify it passes** - -``` -uv run pytest tests/test_protocols/ -k "test_ephemeral_store_protocol" -v -``` -Expected: PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/protocols/node_protocols.py src/orcapod/protocols/pipeline_protocols.py tests/ -git commit -m "feat(protocols): add set_ephemeral_store to FunctionNodeProtocol, OperatorNodeProtocol, PipelineProtocol" -``` - ---- - -## Task 4: No-op `set_ephemeral_store` on base node classes - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (`FunctionNodeBase`, lines ~88-160) -- Modify: `src/orcapod/core/nodes/operator_node.py` (`OperatorNodeBase`, lines ~61-120) - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_core/function_pod/test_ephemeral_result.py (create new file) -"""Tests for FunctionJobNode ephemeral result store — ITL-507.""" -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.datagrams import Data, Tag -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes import FunctionNode -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.sources import ArrowTableSource -from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.databases import InMemoryArrowDatabase -from orcapod.types import NodeConfig - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def double(x: int) -> int: - return x * 2 - - -def _make_pod(config: NodeConfig | None = None): - pf = PythonDataFunction(double, output_keys="result") - return FunctionPod(pf, config=config) - - -def _make_stream(rows: list[dict], tag_columns: list[str] | None = None) -> ArrowTableStream: - if tag_columns is None: - tag_columns = ["id"] - table = pa.table( - {k: pa.array([r[k] for r in rows], type=pa.int64()) for k in rows[0]} - ) - source = ArrowTableSource(table, tag_columns=tag_columns, source_id="test_src", infer_nullable=True) - return source - - -def _make_node(stream, pipeline_db=None, result_db=None, ephemeral_result: bool = False): - """Create a FunctionJobNode with given DB configuration.""" - cfg = NodeConfig(ephemeral_result=ephemeral_result) - pod = _make_pod(config=cfg) - if pipeline_db is None: - pipeline_db = InMemoryArrowDatabase() - return FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db if result_db is not None else pipeline_db, - ), pipeline_db - - -# --------------------------------------------------------------------------- -# Task 4 test: no-op set_ephemeral_store on blueprint node classes -# --------------------------------------------------------------------------- - -class TestNoOpSetEphemeralStore: - def test_function_node_has_set_ephemeral_store(self): - """FunctionNode (blueprint) must have set_ephemeral_store as a no-op.""" - stream = _make_stream([{"id": 0, "x": 10}]) - pf = PythonDataFunction(double, output_keys="result") - pod = FunctionPod(pf) - node = FunctionNode(function_pod=pod, input_stream=stream) - store = InMemoryArrowDatabase() - # Must not raise - node.set_ephemeral_store(store) - node.set_ephemeral_store(None) - - def test_operator_node_has_set_ephemeral_store(self): - """OperatorNode (blueprint) must have set_ephemeral_store as a no-op.""" - from orcapod.core.nodes import OperatorNode - from orcapod.core.operators import Join - - stream_a = _make_stream([{"id": 0, "x": 10}]) - stream_b = _make_stream([{"id": 0, "y": 20}]) - op = Join() - node = OperatorNode(operator=op, input_streams=(stream_a, stream_b)) - store = InMemoryArrowDatabase() - # Must not raise - node.set_ephemeral_store(store) - node.set_ephemeral_store(None) -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestNoOpSetEphemeralStore -v -``` -Expected: FAIL with `AttributeError: 'FunctionNode' object has no attribute 'set_ephemeral_store'` - -- [ ] **Step 3: Add no-op to `FunctionNodeBase`** - -In `src/orcapod/core/nodes/function_node.py`, add to `FunctionNodeBase` after the `__init__` method (and before `# ------------------------------------------------------------------`). Add `TYPE_CHECKING` import for `InMemoryArrowDatabase` at the top of the file — add it inside the existing `if TYPE_CHECKING:` block: - -```python -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - import pyarrow.compute as pc - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase -``` - -Then add the method to `FunctionNodeBase`: - -```python -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """No-op for blueprint nodes — only ``FunctionJobNode`` uses ephemeral stores.""" -``` - -- [ ] **Step 4: Add no-op to `OperatorNodeBase`** - -In `src/orcapod/core/nodes/operator_node.py`, add to `OperatorNodeBase` similarly. First add to `TYPE_CHECKING` block: - -```python -if TYPE_CHECKING: - import pyarrow as pa - import pyarrow.compute as pc - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol -``` - -Then add the method to `OperatorNodeBase`: - -```python -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """No-op — operator nodes do not support ephemeral result storage in v1 (ITL-509).""" -``` - -- [ ] **Step 5: Run test to verify it passes** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestNoOpSetEphemeralStore -v -``` -Expected: PASS - -- [ ] **Step 6: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py src/orcapod/core/nodes/operator_node.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(nodes): add no-op set_ephemeral_store to FunctionNodeBase and OperatorNodeBase" -``` - ---- - -## Task 5: `FunctionJobNode` — ephemeral slots and `set_ephemeral_store` override - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (`FunctionJobNode`, lines ~677-770) - -- [ ] **Step 1: Add test** - -Append to `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -class TestSetEphemeralStore: - def test_set_ephemeral_store_assigns_store(self): - """set_ephemeral_store(store) assigns the ephemeral_result_store attribute.""" - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - store = InMemoryArrowDatabase() - node.set_ephemeral_store(store) - assert node.ephemeral_result_store is store - - def test_set_ephemeral_store_none_detaches(self): - """set_ephemeral_store(None) removes the ephemeral store.""" - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - store = InMemoryArrowDatabase() - node.set_ephemeral_store(store) - node.set_ephemeral_store(None) - assert node.ephemeral_result_store is None -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestSetEphemeralStore -v -``` -Expected: FAIL with `AttributeError: 'FunctionJobNode' object has no attribute 'ephemeral_result_store'` - -- [ ] **Step 3: Add slots and override to `FunctionJobNode`** - -In `FunctionJobNode.__init__` (around line 706), add after the `self._pipeline_database` initialization: - -```python -# Ephemeral result store (None until set_ephemeral_store() is called by the pipeline) -self.ephemeral_result_store: "InMemoryArrowDatabase | None" = None -self._ephemeral_cached_pod: CachedFunctionPod | None = None -``` - -After `FunctionJobNode.attach_databases` (around line 767), add: - -```python -# ------------------------------------------------------------------ -# set_ephemeral_store — override (FunctionJobNode has real behavior) -# ------------------------------------------------------------------ - -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """Assign or remove the ephemeral result store. - - When *store* is not ``None``, creates a ``CachedFunctionPod`` backed by - *store* so that ephemeral writes use the same format as persistent writes. - When *store* is ``None``, clears both the store and the ephemeral pod. - - Args: - store: The ``InMemoryArrowDatabase`` to use for ephemeral result - storage, or ``None`` to detach and revert to persistent-only writes. - """ - self.ephemeral_result_store = store - if store is not None and self._function_pod is not None: - self._ephemeral_cached_pod = CachedFunctionPod( - self._function_pod, - result_database=store, - ) - else: - self._ephemeral_cached_pod = None -``` - -- [ ] **Step 4: Run test to verify it passes** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestSetEphemeralStore -v -``` -Expected: PASS - -- [ ] **Step 5: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(function_node): add ephemeral_result_store slot and set_ephemeral_store override to FunctionJobNode" -``` - ---- - -## Task 6: `add_pipeline_record` — add `is_ephemeral` and enforce `skip_cache_lookup=True` in Phase 2 - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines ~1267-1341) - -This task does two things: -1. Adds `is_ephemeral: bool = False` parameter and stores `IS_EPHEMERAL_COL` in the tag table row. -2. Updates the call-site in `_process_data_internal` to always use `skip_cache_lookup=True` (fixing the potential infinite-miss-cycle for both persistent and ephemeral paths — see spec §3 "Recompute-after-miss write strategy"). - -- [ ] **Step 1: Add test for IS_EPHEMERAL_COL written to pipeline DB** - -Append to `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -class TestAddPipelineRecord: - def test_is_ephemeral_false_written_to_pipeline_db(self): - """add_pipeline_record(is_ephemeral=False) stores IS_EPHEMERAL_COL = False.""" - from orcapod.system_constants import constants - - stream = _make_stream([{"id": 0, "x": 10}]) - node, db = _make_node(stream) - results = node.execute(stream) - assert len(results) == 1 - - all_records = db.get_all_records(node.node_identity_path) - assert all_records is not None - assert constants.IS_EPHEMERAL_COL in all_records.column_names - vals = all_records.column(constants.IS_EPHEMERAL_COL).to_pylist() - assert all(v is False for v in vals) - - def test_is_ephemeral_true_written_to_pipeline_db(self): - """When ephemeral_result=True, IS_EPHEMERAL_COL=True is stored in the tag table.""" - from orcapod.system_constants import constants - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - ephemeral_store = InMemoryArrowDatabase() - node, _ = _make_node(stream, pipeline_db=pipeline_db, ephemeral_result=True) - node.set_ephemeral_store(ephemeral_store) - - results = node.execute(stream) - assert len(results) == 1 - - all_records = pipeline_db.get_all_records(node.node_identity_path) - assert all_records is not None - assert constants.IS_EPHEMERAL_COL in all_records.column_names - vals = all_records.column(constants.IS_EPHEMERAL_COL).to_pylist() - assert all(v is True for v in vals) -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestAddPipelineRecord -v -``` -Expected: FAIL — `IS_EPHEMERAL_COL` not in pipeline DB columns. - -- [ ] **Step 3: Update `add_pipeline_record` signature and body** - -In `src/orcapod/core/nodes/function_node.py`, edit `add_pipeline_record` (lines ~1267-1341): - -Change the signature to add `is_ephemeral: bool = False`: - -```python -def add_pipeline_record( - self, - tag: TagProtocol, - input_data: DataProtocol, - data_record_id: uuid.UUID, - computed: bool, - skip_cache_lookup: bool = False, - is_ephemeral: bool = False, -) -> None: - """Add a pipeline record to the database for a processed data. - - The pipeline record stores: - - Tag columns (including system tags) - - All source columns of the input data (provenance, not data) - - Output data record ID (for joining with result records) - - Whether the result is stored in the ephemeral store - - Input data data context key - - Whether the result was freshly computed or cached - """ -``` - -In the `meta_table` construction (around line 1311), add `IS_EPHEMERAL_COL`: - -```python -meta_table = pa.table( - { - constants.DATA_RECORD_ID: pa.array( - [data_record_id.bytes], type=pa.large_binary() - ), - constants.NODE_CONTENT_HASH_COL: pa.array( - [self.content_hash().to_string()], type=pa.large_string() - ), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array( - [input_data.data_context_key], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array( - [computed], type=pa.bool_() - ), - constants.IS_EPHEMERAL_COL: pa.array( - [is_ephemeral], type=pa.bool_() - ), - } -) -``` - -- [ ] **Step 4: Update `_process_data_internal` call-site (persistent path)** - -In `_process_data_internal` (lines ~1099-1143), update the existing `add_pipeline_record` call to add `skip_cache_lookup=True`: - -```python -# OLD: -self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, -) - -# NEW: -self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, -) -``` - -- [ ] **Step 5: Run test to verify it passes** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestAddPipelineRecord -v -``` -Expected: PASS - -- [ ] **Step 6: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(function_node): add is_ephemeral column to pipeline records; enforce skip_cache_lookup=True in Phase 2" -``` - ---- - -## Task 7: `_fetch_joined_records` — two-store logic - -This is the largest structural change. `_fetch_joined_records` (lines ~1437-1503) is restructured from a single-store join into a two-store join with anti-join priority merge. - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py:1437-1503` - -- [ ] **Step 1: Add tests for the two-store logic** - -Append to `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -class TestBulkResolution: - def test_ephemeral_false_unchanged(self): - """ephemeral_result=False: execute() behaves identically to current implementation.""" - stream = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) - node, _ = _make_node(stream) - results = node.execute(stream) - assert len(results) == 2 - vals = {tag.as_dict()["id"]: data.as_dict()["result"] for tag, data in results} - assert vals == {0: 20, 1: 40} - - def test_ephemeral_result_written_to_memory_not_persistent_db(self): - """With ephemeral_result=True, persistent DB has no result rows; ephemeral store does.""" - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - ephemeral_store = InMemoryArrowDatabase() - cfg = NodeConfig(ephemeral_result=True) - pod = _make_pod(config=cfg) - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db, - ) - node.set_ephemeral_store(ephemeral_store) - results = node.execute(stream) - - assert len(results) == 1 - assert results[0][1].as_dict()["result"] == 20 - - # Persistent result DB must be empty (no writes there) - from orcapod.core.cached_function_pod import CachedFunctionPod - eph_cache = node._ephemeral_cached_pod - assert eph_cache is not None - assert eph_cache.result_database.get_all_records(eph_cache.record_path) is not None - assert result_db.get_all_records(node._cached_function_pod.record_path) is None - - def test_within_session_ephemeral_hit(self): - """Same node called twice: second call hits ephemeral store — no recomputation.""" - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - ephemeral_store = InMemoryArrowDatabase() - pf = PythonDataFunction(counting_double, output_keys="result") - cfg = NodeConfig(ephemeral_result=True) - pod = FunctionPod(pf, config=cfg) - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node.set_ephemeral_store(ephemeral_store) - - node.execute(stream) - assert call_count["n"] == 1 - - # Second execution — same entry_id — must hit cache - node._cached_output_datas.clear() # clear in-memory cache to force DB lookup - node.execute(stream) - assert call_count["n"] == 1 # function must NOT have been called again - - def test_cross_session_miss_recomputes(self): - """Fresh InMemoryArrowDatabase (new session): ephemeral miss triggers recomputation.""" - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - - # Session 1: execute with ephemeral store - pf = PythonDataFunction(counting_double, output_keys="result") - cfg = NodeConfig(ephemeral_result=True) - pod = FunctionPod(pf, config=cfg) - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node.set_ephemeral_store(InMemoryArrowDatabase()) - node.execute(stream) - assert call_count["n"] == 1 - - # Session 2: fresh in-memory node with a fresh ephemeral store - # The pipeline_db still has the tag entry (IS_EPHEMERAL=True), but the - # ephemeral store is fresh — this should trigger a cache miss and recompute. - pf2 = PythonDataFunction(counting_double, output_keys="result") - pod2 = FunctionPod(pf2, config=cfg) - node2 = FunctionJobNode( - function_pod=pod2, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node2.set_ephemeral_store(InMemoryArrowDatabase()) # fresh store - node2.execute(stream) - assert call_count["n"] == 2 # recomputed - - def test_persistent_hit_served_when_ephemeral_true(self): - """A persistent result is still served from cache when ephemeral store is also set. - - Verifies that attaching an ephemeral store doesn't break Phase 1's persistent join. - The node uses ephemeral_result=False (default), so Phase 2 writes to persistent DB. - Phase 1 with two-store logic must still find the persistent result. - """ - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - pf = PythonDataFunction(counting_double, output_keys="result") - pod = FunctionPod(pf) # ephemeral_result=False (default) - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - # Attach an ephemeral store — must NOT break persistent reads - node.set_ephemeral_store(InMemoryArrowDatabase()) - - # Run 1: writes to persistent DB (ephemeral_result=False) - results = node.execute(stream) - assert len(results) == 1 - assert results[0][1].as_dict()["result"] == 20 - assert call_count["n"] == 1 - - # Clear in-memory cache to force DB lookup on Run 2 - node._cached_output_datas.clear() - - # Run 2: Phase 1 must find persistent result — no recompute - results2 = node.execute(stream) - assert len(results2) == 1 - assert results2[0][1].as_dict()["result"] == 20 - assert call_count["n"] == 1 # NOT recomputed - - def test_bulk_resolution_mixed_stores(self): - """Tag table has both persistent and ephemeral entries; both resolve correctly.""" - stream_a = _make_stream([{"id": 0, "x": 10}]) - stream_b = _make_stream([{"id": 1, "x": 20}]) - pipeline_db = InMemoryArrowDatabase() - - # id=0 → persistent - node_p, _ = _make_node(stream_a, pipeline_db=pipeline_db, ephemeral_result=False) - node_p.execute(stream_a) - - # id=1 → ephemeral, reusing same pipeline_db - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - ephemeral_store = InMemoryArrowDatabase() - pf = PythonDataFunction(counting_double, output_keys="result") - cfg = NodeConfig(ephemeral_result=True) - pod = FunctionPod(pf, config=cfg) - node_e = FunctionJobNode( - function_pod=pod, - input_stream=stream_b, - pipeline_database=pipeline_db, - ) - node_e.set_ephemeral_store(ephemeral_store) - node_e.execute(stream_b) - - # Now load both results via a combined stream - stream_both = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) - node_both = FunctionJobNode( - function_pod=pod, - input_stream=stream_both, - pipeline_database=pipeline_db, - ) - node_both.set_ephemeral_store(ephemeral_store) - node_both._cached_output_datas.clear() - - results = node_both.execute(stream_both) - assert len(results) == 2 - vals = {tag.as_dict()["id"]: data.as_dict()["result"] for tag, data in results} - assert vals == {0: 20, 1: 40} - - def test_persistent_result_outcompetes_ephemeral(self): - """When both persistent and ephemeral entries exist for the same entry_id, persistent wins. - - This tests the anti-join merge: a tag table row with IS_EPHEMERAL=True that shares - an entry_id with an IS_EPHEMERAL=False row should be excluded from available_results. - """ - # Use a single node to write both a persistent AND an ephemeral entry for id=0. - # We do this by: - # 1. Execute with ephemeral store attached but ephemeral_result=False → writes persistent entry - # 2. Manually insert an ephemeral tag-table row + ephemeral store result to simulate - # the scenario where both exist (e.g. after a recovery run). - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - ephemeral_store = InMemoryArrowDatabase() - - # Write a persistent entry - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - pf = PythonDataFunction(counting_double, output_keys="result") - pod = FunctionPod(pf) - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node.set_ephemeral_store(ephemeral_store) - node.execute(stream) # writes IS_EPHEMERAL=False persistent entry - assert call_count["n"] == 1 - - # Simulate an ephemeral entry also existing for the same input - # by calling add_pipeline_record directly with is_ephemeral=True. - # (This represents the "recovery scenario" from the spec.) - import uuid as _uuid - tag = Tag({"id": 0}) - data = Data({"x": 10}) - fake_uuid = _uuid.UUID(int=0) # dummy UUID, no actual result stored - node.add_pipeline_record( - tag, data, - data_record_id=fake_uuid, - computed=True, - skip_cache_lookup=True, - is_ephemeral=True, - ) - - # Clear in-memory cache to force fresh DB lookup - node._cached_output_datas.clear() - - # Phase 1: both persistent (result=20) and ephemeral (fake UUID, no actual result) - # entries exist. Persistent should win — ephemeral entry excluded by anti-join. - # Since fake_uuid has no actual result in ephemeral_store, the ephemeral join - # produces no row for it. The persistent join wins, and result=20 is returned. - results = node.execute(stream) - assert len(results) == 1 - assert results[0][1].as_dict()["result"] == 20 - assert call_count["n"] == 1 # NOT recomputed -``` - -- [ ] **Step 2: Run the test to verify it fails** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestBulkResolution -v -``` -Expected: multiple failures — `IS_EPHEMERAL_COL` not used in `_fetch_joined_records`, result written to wrong store, etc. - -- [ ] **Step 3: Restructure `_fetch_joined_records`** - -Replace the entire body of `_fetch_joined_records` in `function_node.py` (lines ~1437-1503): - -```python -def _fetch_joined_records( - self, - entry_ids: list[bytes] | None = None, -) -> _JoinedRecords | None: - """Internal primitive: fetch both DBs and inner-join, supporting two stores. - - Fetches ``taginfo`` from the pipeline database, partitions rows by - ``IS_EPHEMERAL_COL`` into persistent and ephemeral groups, performs two - independent inner joins (one per store), merges with persistent priority - via an anti-join, and returns the combined result. - - Persistent miss rows (tag entry with no matching result DB row) emit a - WARNING-level log. Ephemeral miss rows (cross-session miss) are silently - dropped. - - If ``entry_ids`` is provided, the result is filtered to matching - ``_PIPELINE_ENTRY_ID_COL`` values before conversion to Arrow. - - Args: - entry_ids: If given, return only rows whose - ``_PIPELINE_ENTRY_ID_COL`` value is in this list. - If ``None``, return all rows. - - Returns: - A ``_JoinedRecords`` whose ``table`` always includes a - ``_PIPELINE_ENTRY_ID_COL`` column, or ``None`` when either - the pipeline database or cached function pod is absent. A 0-row - table (not ``None``) is returned when both fetches succeed but - no matching rows exist — callers check ``num_rows`` themselves. - """ - if self._cached_function_pod is None or self._pipeline_database is None: - return None - - taginfo = self._pipeline_database.get_all_records( - self.node_identity_path, - record_id_column=_PIPELINE_ENTRY_ID_COL, - ) - - if taginfo is None: - return None - - taginfo_columns = tuple(taginfo.column_names) - taginfo = self._filter_by_content_hash(taginfo) - taginfo_schema = taginfo.schema - - is_ephemeral_col = constants.IS_EPHEMERAL_COL - taginfo_df = pl.from_arrow(taginfo) - - # Partition by IS_EPHEMERAL_COL (backward-compat: missing col → all persistent) - if is_ephemeral_col in taginfo.column_names: - persistent_taginfo_df = taginfo_df.filter( - ~pl.col(is_ephemeral_col).fill_null(False) - ) - ephemeral_taginfo_df = taginfo_df.filter( - pl.col(is_ephemeral_col).fill_null(False) - ) - else: - persistent_taginfo_df = taginfo_df - ephemeral_taginfo_df = pl.DataFrame() - - # ------------------------------------------------------------------ - # Persistent join - # ------------------------------------------------------------------ - results_schema = None - persistent_df = pl.DataFrame() - if persistent_taginfo_df.height > 0: - results = self._cached_function_pod.result_database.get_all_records( - self._cached_function_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) - if results is None: - # Tag table has persistent entries but result DB is empty — data loss - for row_dict in persistent_taginfo_df.to_dicts(): - logger.warning( - "Pipeline DB entry '%s' has no match in persistent result DB " - "— data may have been deleted externally. " - "This input will be recomputed.", - row_dict.get(_PIPELINE_ENTRY_ID_COL), - ) - else: - results_schema = results.schema - full_persistent_df = persistent_taginfo_df.join( - pl.from_arrow(results), - on=constants.DATA_RECORD_ID, - how="inner", - ) - # Warn about persistent tag rows that found no match in the result DB - if full_persistent_df.height < persistent_taginfo_df.height: - matched_ids = set( - full_persistent_df.select(_PIPELINE_ENTRY_ID_COL) - .to_series() - .to_list() - ) - for row_dict in persistent_taginfo_df.to_dicts(): - if row_dict[_PIPELINE_ENTRY_ID_COL] not in matched_ids: - logger.warning( - "Pipeline DB entry '%s' has no match in persistent result DB " - "— data may have been deleted externally. " - "This input will be recomputed.", - row_dict[_PIPELINE_ENTRY_ID_COL], - ) - persistent_df = full_persistent_df - - # ------------------------------------------------------------------ - # Ephemeral join - # ------------------------------------------------------------------ - ephemeral_df = pl.DataFrame() - if ephemeral_taginfo_df.height > 0 and self._ephemeral_cached_pod is not None: - eph_results = self._ephemeral_cached_pod.result_database.get_all_records( - self._ephemeral_cached_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) - if eph_results is not None: - # Cross-session miss → eph_results is None → silently drop (expected) - if results_schema is None: - results_schema = eph_results.schema - ephemeral_df = ephemeral_taginfo_df.join( - pl.from_arrow(eph_results), - on=constants.DATA_RECORD_ID, - how="inner", - ) - - # ------------------------------------------------------------------ - # Merge with persistent priority (anti-join + concat) - # ------------------------------------------------------------------ - if ephemeral_df.height > 0 and persistent_df.height > 0: - ephemeral_only_df = ephemeral_df.join( - persistent_df.select([_PIPELINE_ENTRY_ID_COL]), - on=_PIPELINE_ENTRY_ID_COL, - how="anti", - ) - merged_df = pl.concat([persistent_df, ephemeral_only_df], how="diagonal") - elif ephemeral_df.height > 0: - merged_df = ephemeral_df - elif persistent_df.height > 0: - merged_df = persistent_df - else: - # No results found in either store — return empty table preserving schema - if taginfo.num_rows == 0 or results_schema is None: - # Both tables empty — return 0-row table from taginfo schema - empty_table = taginfo.slice(0, 0) - return _JoinedRecords(table=empty_table, taginfo_columns=taginfo_columns) - # We have tag rows but no matching results — also return empty - empty_table = taginfo.slice(0, 0) - return _JoinedRecords(table=empty_table, taginfo_columns=taginfo_columns) - - # Apply entry_id filter if requested - if entry_ids is not None: - merged_df = merged_df.filter( - pl.col(_PIPELINE_ENTRY_ID_COL).is_in(entry_ids) - ) - - joined = merged_df.to_arrow() - if results_schema is not None: - joined = arrow_utils.restore_schema_nullability( - joined, taginfo_schema, results_schema - ) - return _JoinedRecords(table=joined, taginfo_columns=taginfo_columns) -``` - -Note: `pl.from_arrow()` is used instead of `pl.DataFrame()` for creating Polars DataFrames from Arrow tables — this is cleaner than `pl.DataFrame(arrow_table)`. Verify this API works in the Polars version used by this project. If `pl.from_arrow` is not available, use `pl.DataFrame(arrow_table)` (the original form). - -- [ ] **Step 4: Run test to verify it passes** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestBulkResolution -v -``` -Expected: PASS for all tests in this class. Some tests may still fail if `_process_data_internal` hasn't been updated yet (Task 8). - -- [ ] **Step 5: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all existing tests still pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(function_node): restructure _fetch_joined_records for two-store ephemeral/persistent join" -``` - ---- - -## Task 8: `_process_data_internal` — ephemeral write path - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py:1099-1143` - -- [ ] **Step 1: Add tests for the ephemeral write path** - -Append to `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -class TestEphemeralWritePath: - def test_store_not_assigned_raises(self): - """ephemeral_result=True but ephemeral_result_store=None raises RuntimeError.""" - stream = _make_stream([{"id": 0, "x": 10}]) - cfg = NodeConfig(ephemeral_result=True) - pod = _make_pod(config=cfg) - pipeline_db = InMemoryArrowDatabase() - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - # set_ephemeral_store never called → ephemeral_result_store is None - with pytest.raises(RuntimeError, match="ephemeral_result=True"): - node.execute(stream) - - def test_ephemeral_only_node(self): - """result_database=None, ephemeral_result=True: end-to-end works.""" - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - ephemeral_store = InMemoryArrowDatabase() - cfg = NodeConfig(ephemeral_result=True) - pod = _make_pod(config=cfg) - # Pass pipeline_database but no result_database — ephemeral only - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node.set_ephemeral_store(ephemeral_store) - results = node.execute(stream) - assert len(results) == 1 - assert results[0][1].as_dict()["result"] == 20 - - def test_recompute_after_ephemeral_miss_no_infinite_cycle(self): - """Cross-session ephemeral miss → recomputed → served on next call (no cycle).""" - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - - # Session 1 - pf = PythonDataFunction(counting_double, output_keys="result") - cfg = NodeConfig(ephemeral_result=True) - pod = FunctionPod(pf, config=cfg) - node1 = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node1.set_ephemeral_store(InMemoryArrowDatabase()) - node1.execute(stream) - assert call_count["n"] == 1 - - # Session 2: fresh store → miss → recompute - pf2 = PythonDataFunction(counting_double, output_keys="result") - pod2 = FunctionPod(pf2, config=cfg) - ephemeral2 = InMemoryArrowDatabase() - node2 = FunctionJobNode( - function_pod=pod2, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node2.set_ephemeral_store(ephemeral2) - node2.execute(stream) - assert call_count["n"] == 2 - - # Session 3: same ephemeral store as session 2 → hit → no recompute - pf3 = PythonDataFunction(counting_double, output_keys="result") - pod3 = FunctionPod(pf3, config=cfg) - node3 = FunctionJobNode( - function_pod=pod3, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node3.set_ephemeral_store(ephemeral2) # reuse session 2's store - node3._cached_output_datas.clear() - node3.execute(stream) - assert call_count["n"] == 2 # NOT recomputed -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestEphemeralWritePath -v -``` -Expected: `test_store_not_assigned_raises` fails (no RuntimeError), others may also fail. - -- [ ] **Step 3: Update `_process_data_internal`** - -In `src/orcapod/core/nodes/function_node.py`, replace the body of `_process_data_internal` (lines ~1099-1143): - -```python -def _process_data_internal( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - """Core compute + persist + cache. - - Used by ``execute_data`` and ``execute``. - Stores result in ``_cached_output_datas`` keyed by entry_id. - Exceptions propagate to the caller — no error handling here. - - When ``node_config.ephemeral_result=True``: - - Uses ``_ephemeral_cached_pod`` for both compute and storage. - - Raises ``RuntimeError`` if no ephemeral store has been set. - - Writes the pipeline record with ``is_ephemeral=True`` and - ``skip_cache_lookup=True`` (append strategy for recompute-after-miss). - - When ``node_config.ephemeral_result=False`` (default): - - Uses ``_cached_function_pod`` (persistent DB) or raw function pod. - - Writes the pipeline record with ``skip_cache_lookup=True`` (prevents - infinite miss cycle when a stale pipeline entry exists). - - Returns: - A ``(tag, output_data)`` 2-tuple. - """ - node_config = self._function_pod.config if self._function_pod is not None else None - ephemeral_result = ( - node_config.ephemeral_result - if node_config is not None - else False - ) - - if ephemeral_result: - if self._ephemeral_cached_pod is None: - raise RuntimeError( - f"FunctionJobNode '{self.label}' has ephemeral_result=True but no " - "ephemeral store has been assigned. Call set_ephemeral_store() with " - "an InMemoryArrowDatabase before executing this node." - ) - tag_out, output_data = self._ephemeral_cached_pod.process_data( - tag, data, logger=logger - ) - if output_data is not None: - result_computed = bool( - output_data.get_meta_value( - self._ephemeral_cached_pod.RESULT_COMPUTED_FLAG, True - ) - ) - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, - is_ephemeral=True, - ) - elif self._cached_function_pod is not None: - tag_out, output_data = self._cached_function_pod.process_data( - tag, data, logger=logger - ) - if output_data is not None: - result_computed = bool( - output_data.get_meta_value( - self._cached_function_pod.RESULT_COMPUTED_FLAG, False - ) - ) - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, - ) - else: - tag_out, output_data = self._function_pod.process_data( - tag, data, logger=logger - ) - - # Store by entry_id and invalidate derived caches - entry_id = self.compute_pipeline_entry_id(tag, data) - self._cached_output_datas[entry_id] = (tag_out, output_data) - self._cached_output_table = None - self._cached_content_hash_column = None - - return tag_out, output_data -``` - -Note: `self._function_pod.config` accesses the `NodeConfig` from the pod. Verify this attribute path by checking `FunctionPod.config` in `src/orcapod/core/function_pod.py`. If the path is different, adjust accordingly (it may be `self._function_pod.node_config` or similar). If `NodeConfig` is not accessible from the pod at this point, add a `self._node_config` slot to `FunctionJobNode.__init__` set from `function_pod.config` at construction time. - -- [ ] **Step 4: Verify NodeConfig access path** - -``` -uv run python -c " -from orcapod.core.function_pod import FunctionPod -from orcapod.core.data_function import PythonDataFunction -from orcapod.types import NodeConfig -pf = PythonDataFunction(lambda x: x, output_keys='y') -cfg = NodeConfig(ephemeral_result=True) -pod = FunctionPod(pf, config=cfg) -print(type(pod.config)) # should print NodeConfig -print(pod.config.ephemeral_result) # should print True -" -``` - -If `pod.config` raises `AttributeError`, find the correct attribute name: - -``` -uv run python -c " -from orcapod.core.function_pod import FunctionPod -from orcapod.core.data_function import PythonDataFunction -pf = PythonDataFunction(lambda x: x, output_keys='y') -pod = FunctionPod(pf) -print([a for a in dir(pod) if 'config' in a.lower() or 'node' in a.lower()]) -" -``` - -Adjust the `ephemeral_result` lookup in `_process_data_internal` to use the correct attribute. - -- [ ] **Step 5: Run test to verify it passes** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestEphemeralWritePath -v -``` -Expected: PASS for all three tests. - -- [ ] **Step 6: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(function_node): implement ephemeral write path in _process_data_internal" -``` - ---- - -## Task 9: `AbstractPipelineBase.set_ephemeral_store` - -**Files:** -- Modify: `src/orcapod/pipeline/base.py` - -- [ ] **Step 1: Add test** - -Append to `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -class TestPipelineInjectsStore: - def test_pipeline_job_set_ephemeral_store_propagates(self): - """PipelineJob.set_ephemeral_store propagates to all compiled nodes.""" - from orcapod.pipeline.job import PipelineJob - - stream = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) - cfg = NodeConfig(ephemeral_result=True) - pod = _make_pod(config=cfg) - - job = PipelineJob(name="test_pipeline") - with job: - output = pod(stream) - - ephemeral_store = InMemoryArrowDatabase() - job.set_ephemeral_store(ephemeral_store) - - # Every function node in the compiled pipeline should now have the store - for label, node in job.function_pods.items(): - assert node.ephemeral_result_store is ephemeral_store, ( - f"Node '{label}' did not receive the ephemeral store" - ) - - def test_pipeline_job_set_ephemeral_store_none_detaches(self): - """PipelineJob.set_ephemeral_store(None) detaches the store from all nodes.""" - from orcapod.pipeline.job import PipelineJob - - stream = _make_stream([{"id": 0, "x": 10}]) - cfg = NodeConfig(ephemeral_result=True) - pod = _make_pod(config=cfg) - - job = PipelineJob(name="test_pipeline") - with job: - pod(stream) - - store = InMemoryArrowDatabase() - job.set_ephemeral_store(store) - job.set_ephemeral_store(None) - - for label, node in job.function_pods.items(): - assert node.ephemeral_result_store is None, ( - f"Node '{label}' ephemeral store was not detached" - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestPipelineInjectsStore -v -``` -Expected: FAIL with `AttributeError: 'PipelineJob' object has no attribute 'set_ephemeral_store'` - -- [ ] **Step 3: Add `set_ephemeral_store` to `AbstractPipelineBase`** - -In `src/orcapod/pipeline/base.py`, add the `InMemoryArrowDatabase` import inside the `TYPE_CHECKING` block: - -```python -if TYPE_CHECKING: - import networkx as nx - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase -``` - -Add the method to `AbstractPipelineBase` after the `operator_pods` property (around line 137): - -```python -def set_ephemeral_store(self, store: "InMemoryArrowDatabase | None") -> None: - """Assign or remove the ephemeral result store for all compiled nodes. - - Propagates *store* to every compiled node in the pipeline by calling - ``node.set_ephemeral_store(store)`` on each. Nodes that do not support - ephemeral storage (e.g. operator nodes in v1, blueprint function nodes) - treat this as a no-op. - - Pass ``None`` to detach the ephemeral store from all nodes, reverting - them to persistent-only behaviour for subsequent writes. - - Args: - store: The ``InMemoryArrowDatabase`` to use for ephemeral result - storage, or ``None`` to detach from all nodes. - """ - for node in self._nodes.values(): - node.set_ephemeral_store(store) -``` - -- [ ] **Step 4: Run test to verify it passes** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestPipelineInjectsStore -v -``` -Expected: PASS - -- [ ] **Step 5: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/base.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(pipeline): add set_ephemeral_store to AbstractPipelineBase propagating to all nodes" -``` - ---- - -## Task 10: Remaining tests — persistent miss warning and append-after-miss - -- [ ] **Step 1: Add remaining tests** - -Append to `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -class TestPersistentMissWarning: - def test_persistent_miss_warns_and_recomputes(self, caplog): - """Tag table has a regular record_id but persistent DB was trimmed: WARNING emitted.""" - import logging - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - - # Write a persistent record - node, _ = _make_node(stream, pipeline_db=pipeline_db, result_db=result_db, ephemeral_result=False) - node.execute(stream) - - # Wipe the result DB to simulate data loss - result_db._tables.clear() - result_db._pending_batches.clear() - - # Recreate node with same pipeline_db (tag entry still there) but empty result_db - node2, _ = _make_node(stream, pipeline_db=pipeline_db, result_db=result_db, ephemeral_result=False) - - with caplog.at_level(logging.WARNING, logger="orcapod.core.nodes.function_node"): - results = node2.execute(stream) - - assert len(results) == 1 # recomputed - assert any("has no match in persistent result DB" in msg for msg in caplog.messages) - - def test_recompute_after_persistent_miss_appends_new_pipeline_record(self): - """After persistent miss and recompute, tag table has two rows; next call hits.""" - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - - # Session 1: write persistent record - pf = PythonDataFunction(counting_double, output_keys="result") - pod = FunctionPod(pf) - node1 = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db, - ) - node1.execute(stream) - assert call_count["n"] == 1 - - # Simulate data loss - result_db._tables.clear() - result_db._pending_batches.clear() - - # Session 2: miss → recompute → appends new record to pipeline_db - pf2 = PythonDataFunction(counting_double, output_keys="result") - pod2 = FunctionPod(pf2) - node2 = FunctionJobNode( - function_pod=pod2, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db, - ) - node2.execute(stream) - assert call_count["n"] == 2 # recomputed - - # Session 3: tag table now has two rows (stale + new); inner join resolves correctly - pf3 = PythonDataFunction(counting_double, output_keys="result") - pod3 = FunctionPod(pf3) - node3 = FunctionJobNode( - function_pod=pod3, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db, - ) - node3._cached_output_datas.clear() - node3.execute(stream) - assert call_count["n"] == 2 # NOT recomputed — new row was found - - -class TestEphemeralOnlyNode: - def test_ephemeral_only_node_no_persistent_db(self): - """NodeConfig(ephemeral_result=True) with no result_database works end-to-end.""" - stream = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) - pipeline_db = InMemoryArrowDatabase() - ephemeral_store = InMemoryArrowDatabase() - cfg = NodeConfig(ephemeral_result=True) - pf = PythonDataFunction(double, output_keys="result") - pod = FunctionPod(pf, config=cfg) - - # No result_database — pipeline_db doubles as both - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - ) - node.set_ephemeral_store(ephemeral_store) - results = node.execute(stream) - - assert len(results) == 2 - vals = {tag.as_dict()["id"]: data.as_dict()["result"] for tag, data in results} - assert vals == {0: 20, 1: 40} -``` - -- [ ] **Step 2: Run new tests to verify they pass (or fail informatively)** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestPersistentMissWarning tests/test_core/function_pod/test_ephemeral_result.py::TestEphemeralOnlyNode -v -``` -Expected: PASS (the WARNING-emitting path was already added in Task 7). - -- [ ] **Step 3: Run the complete ephemeral test file** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py -v -``` -Expected: all tests PASS. - -- [ ] **Step 4: Run full test suite** - -``` -uv run pytest tests/ -x -q -``` -Expected: all pass. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "test(ephemeral_result): add persistent miss warning and append-after-miss tests" -``` - ---- - -## Task 11: Final validation and PR - -- [ ] **Step 1: Run the full test suite one final time** - -``` -uv run pytest tests/ -q -``` -Expected: all pass, no failures. - -- [ ] **Step 2: Verify IS_EPHEMERAL_COL does not appear in any `get_all_records` user-facing output** - -``` -uv run python -c " -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.types import NodeConfig -import pyarrow as pa - -table = pa.table({'id': pa.array([0], type=pa.int64()), 'x': pa.array([10], type=pa.int64())}) -source = ArrowTableSource(table, tag_columns=['id'], source_id='src', infer_nullable=True) - -pf = PythonDataFunction(lambda x: x * 2, output_keys='result') -cfg = NodeConfig(ephemeral_result=True) -pod = FunctionPod(pf, config=cfg) -pipeline_db = InMemoryArrowDatabase() -node = FunctionJobNode(function_pod=pod, input_stream=source, pipeline_database=pipeline_db) -node.set_ephemeral_store(InMemoryArrowDatabase()) -node.execute(source) -records = node.get_all_records() -print('columns:', records.column_names) -print('result:', records.to_pydict()) -" -``` - -Expected: `result` column present, `__is_ephemeral` NOT in `records.column_names` (it's an internal column and should be dropped by `get_all_records`' ColumnConfig filtering, since it starts with `META_PREFIX`). - -If `__is_ephemeral` does appear in `get_all_records` output, add it to the `drop_cols` list in `_load_cached_entries` (around line 1560): - -```python -drop_cols = [ - c - for c in joined.column_names - if c.startswith(constants.META_PREFIX) - or c == constants.NODE_CONTENT_HASH_COL -] -``` - -Since `IS_EPHEMERAL_COL = f"{META_PREFIX}is_ephemeral"`, it starts with `META_PREFIX` and will be automatically excluded. No change needed. - -- [ ] **Step 3: Push branch and open PR** - -```bash -git push -u origin agent-kurodo/itl-507-ephemeral-function-pod-output -gh pr create \ - --title "feat(ITL-507): ephemeral function-pod output v1" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Adds `NodeConfig.ephemeral_result: bool = False` field -- New `set_ephemeral_store(store: InMemoryArrowDatabase | None)` method on `FunctionNodeProtocol`, `OperatorNodeProtocol`, `PipelineProtocol`, and `AbstractPipelineBase` -- `FunctionJobNode` gains `ephemeral_result_store` slot and two-store read/write logic: `_fetch_joined_records` now performs two independent Polars inner-joins (one per store) merged with an anti-join that gives persistent results priority -- `IS_EPHEMERAL` boolean column in the pipeline tag table routes record lookups to the correct store (replaces the "temp:" string prefix from the spec, preserving `DATA_RECORD_ID` as `pa.large_binary()`) -- All Phase 2 writes use `skip_cache_lookup=True` to prevent infinite miss cycles on stale entries (v1 append strategy) -- Persistent DB miss → WARNING log; ephemeral store miss → silently recomputed (expected cross-session behavior) -- 17 test scenarios in `tests/test_core/function_pod/test_ephemeral_result.py` - -## Test plan - -- [ ] `uv run pytest tests/ -q` — all tests pass -- [ ] `test_ephemeral_result_written_to_memory_not_persistent_db` — result goes to ephemeral store only -- [ ] `test_within_session_ephemeral_hit` — second call does not recompute -- [ ] `test_cross_session_miss_recomputes` — fresh store triggers recompute -- [ ] `test_persistent_hit_served_when_ephemeral_true` — persistent cache still works -- [ ] `test_persistent_result_outcompetes_ephemeral` — persistent wins anti-join -- [ ] `test_persistent_miss_warns_and_recomputes` — WARNING log emitted -- [ ] `test_store_not_assigned_raises` — clear RuntimeError message - -Fixes ITL-507 -Related: ITL-508 (indexed recomputation for v0.2.0), ITL-509 (operator ephemeral support) -EOF -)" -``` - ---- - -## Implementation Notes - -### NodeConfig access in `_process_data_internal` - -The code reads `self._function_pod.config` to get `NodeConfig`. Verify this path first (Task 8, Step 4). The config is passed to `FunctionPod(pf, config=cfg)` at construction. If `FunctionPod.config` returns the `NodeConfig`, this is correct. If not, store `NodeConfig` at `FunctionJobNode.__init__` time: - -```python -# In FunctionJobNode.__init__, after super().__init__: -self._node_config: NodeConfig = ( - function_pod.config if hasattr(function_pod, "config") and isinstance(function_pod.config, NodeConfig) - else NodeConfig() -) -``` - -Then use `self._node_config.ephemeral_result` in `_process_data_internal`. - -### Polars API compatibility - -The plan uses `pl.from_arrow(table)` to create a Polars DataFrame from an Arrow table. This is the idiomatic Polars v1+ API. If the project uses an older Polars version, use `pl.DataFrame(table)` instead (same as the existing `_fetch_joined_records` code). Verify with: - -``` -uv run python -c "import polars as pl; print(pl.__version__)" -``` - -### Schema consistency in `_fetch_joined_records` - -The anti-join + concat (`how="diagonal"`) handles the case where IS_EPHEMERAL_COL may differ between persistent and ephemeral rows (it will be `False` vs `True`). `how="diagonal"` pads missing columns with null. If both DataFrames always have IS_EPHEMERAL_COL (which they do since it's in taginfo), use `how="vertical"` instead (stricter, requires identical schemas). - -### Backward compatibility of IS_EPHEMERAL_COL - -Existing pipeline DBs written before this change will not have the IS_EPHEMERAL_COL column. The `_fetch_joined_records` implementation handles this with a fallback: if IS_EPHEMERAL_COL is absent from `taginfo.column_names`, all rows are treated as persistent. This ensures existing cached data remains accessible after the upgrade. diff --git a/superpowers/plans/2026-07-08-itl-471-spiraldb-arrow-metadata-preservation.md b/superpowers/plans/2026-07-08-itl-471-spiraldb-arrow-metadata-preservation.md deleted file mode 100644 index ee8eac306..000000000 --- a/superpowers/plans/2026-07-08-itl-471-spiraldb-arrow-metadata-preservation.md +++ /dev/null @@ -1,1593 +0,0 @@ -# ITL-471: SpiralDB Arrow Metadata Preservation Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Preserve Arrow table- and column-level metadata (including nested struct/list field metadata) across SpiralDB write/read cycles using the native KV store, and replace the blanket extension-type guard in `ConnectorArrowDatabase` with a per-connector `validate_records` hook. - -**Architecture:** Four private helpers in `spiraldb_connector.py` handle recursive metadata serialization/deserialization. `upsert_records` writes metadata to the SpiralDB KV store after each data write; `iter_batches` reads and restores it per scan. `DBConnectorProtocol` gains an abstract `validate_records` method; each connector implements it (rejecting for SQLite/PostgreSQL, no-op for SpiralDB); `ConnectorArrowDatabase` delegates to it instead of applying its own guard. - -**Tech Stack:** PyArrow (`pa.Field`, `pa.DataType`, `pa.struct`, `pa.list_`, `pa.large_list`, `pa.fixed_size_list`), Python `base64` + `json` stdlib, pyspiral `Table.set_metadata` / `Table.get_metadata`, `unittest.mock`. - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/databases/spiraldb_connector.py` | Modify | Add 4 private helpers; modify `upsert_records` and `iter_batches`; add `validate_records` no-op | -| `src/orcapod/protocols/db_connector_protocol.py` | Modify | Add abstract `validate_records` method | -| `src/orcapod/databases/sqlite_connector.py` | Modify | Add rejecting `validate_records` method | -| `src/orcapod/databases/postgresql_connector.py` | Modify | Add rejecting `validate_records` method | -| `src/orcapod/databases/connector_arrow_database.py` | Modify | Replace inline guard (~lines 247–273) with `self._connector.validate_records(records)` | -| `tests/test_databases/test_spiraldb_metadata_helpers.py` | Create | Pure-function tests for the 4 helpers | -| `tests/test_databases/test_spiraldb_connector.py` | Modify | Add `TestUpsertRecordsMetadata`, `TestIterBatchesMetadata`, `TestValidateRecords` | -| `tests/test_databases/test_connector_arrow_database.py` | Modify | Update `MockDBConnector` + `TestExtensionTypeWriteGuard` | -| `tests/test_databases/test_spiraldb_connector_integration.py` | Modify | Add `TestArrowMetadataRoundTrip` | - ---- - -## Implementation Notes (read before starting) - -- **Run all Python via `uv run`** — never call `python` or `pytest` directly. -- **Branch:** `eywalker/itl-471-spiraldbconnector-preserve-arrow-table-and-column-level` -- **`_ARROW_METADATA_KEY = "__arrow_metadata__"`** — the single reserved KV key; define it as a module-level constant in `spiraldb_connector.py`. -- **`validate_records` inheritance:** Connectors do NOT inherit from `DBConnectorProtocol` (structural subtyping only). The abstract method on the Protocol is for documentation/type checking. Each connector must implement it explicitly. `MockDBConnector` in the test file also needs it. -- **`field.with_type(new_type)`** — use this PyArrow method to create a new `pa.Field` with a different type but same name, nullability, and metadata. Check if it exists; if not, use `pa.field(f.name, new_type, f.nullable, f.metadata)`. -- **`pa.fixed_size_list`** — `pa.fixed_size_list(value_type_or_field, list_size)`. Use `arrow_type.list_size` for the size. -- **`pa.map_` children** — `pa.map_` does not support field-level metadata on key/item via constructor. Skip children for map types (no recursion into map key/item fields). -- **Integration tests** are gated on `SPIRAL_INTEGRATION_TESTS=1`. Run unit tests normally; skip integration if no credentials. - ---- - -## Task 1: Checkout branch and run existing tests baseline - -**Files:** (no source changes) - -- [ ] **Step 1: Create and checkout the feature branch** - -```bash -cd /path/to/orcapod-python # your working directory -git checkout -b eywalker/itl-471-spiraldbconnector-preserve-arrow-table-and-column-level -git branch --show-current -``` - -Expected output: `eywalker/itl-471-spiraldbconnector-preserve-arrow-table-and-column-level` - -- [ ] **Step 2: Verify existing tests pass** - -```bash -uv run pytest tests/test_databases/ -v --tb=short -q -``` - -Expected: all pass (integration tests skipped). Note the count — it's the baseline. - ---- - -## Task 2: Serialization helpers — `_serialize_field_meta_tree` and `_serialize_arrow_metadata` - -**Files:** -- Create: `tests/test_databases/test_spiraldb_metadata_helpers.py` -- Modify: `src/orcapod/databases/spiraldb_connector.py` - -### Step 1: Write failing tests for `_serialize_field_meta_tree` - -- [ ] Create `tests/test_databases/test_spiraldb_metadata_helpers.py`: - -```python -"""Pure-function tests for SpiralDB Arrow metadata helpers.""" -from __future__ import annotations - -import base64 -import json - -import pyarrow as pa -import pytest - -from orcapod.databases.spiraldb_connector import ( - _ARROW_METADATA_KEY, - _load_arrow_metadata, - _restore_field, - _serialize_arrow_metadata, - _serialize_field_meta_tree, -) - - -def b64(s: bytes) -> str: - return base64.b64encode(s).decode() - - -# --------------------------------------------------------------------------- -# _serialize_field_meta_tree -# --------------------------------------------------------------------------- - - -class TestSerializeFieldMetaTree: - def test_primitive_with_metadata(self): - field = pa.field("x", pa.int64(), metadata={b"unit": b"meters"}) - result = _serialize_field_meta_tree(field) - assert result == {"meta": {b64(b"unit"): b64(b"meters")}} - - def test_primitive_no_metadata_returns_none(self): - field = pa.field("x", pa.int64()) - assert _serialize_field_meta_tree(field) is None - - def test_struct_with_child_metadata(self): - inner = pa.field("val", pa.float64(), metadata={b"key": b"v"}) - field = pa.field("s", pa.struct([inner])) - result = _serialize_field_meta_tree(field) - assert result == { - "children": {"val": {"meta": {b64(b"key"): b64(b"v")}}} - } - - def test_struct_with_both_own_and_child_metadata(self): - inner = pa.field("val", pa.float64(), metadata={b"k": b"v"}) - field = pa.field("s", pa.struct([inner]), metadata={b"tag": b"yes"}) - result = _serialize_field_meta_tree(field) - assert result == { - "meta": {b64(b"tag"): b64(b"yes")}, - "children": {"val": {"meta": {b64(b"k"): b64(b"v")}}}, - } - - def test_struct_child_no_metadata_not_included(self): - inner_with = pa.field("a", pa.int64(), metadata={b"k": b"v"}) - inner_without = pa.field("b", pa.int64()) - field = pa.field("s", pa.struct([inner_with, inner_without])) - result = _serialize_field_meta_tree(field) - assert "children" in result - assert "a" in result["children"] - assert "b" not in result["children"] - - def test_list_value_field_with_metadata(self): - vf = pa.field("item", pa.int32(), metadata={b"k": b"v"}) - field = pa.field("lst", pa.list_(vf)) - result = _serialize_field_meta_tree(field) - assert result == { - "children": {"item": {"meta": {b64(b"k"): b64(b"v")}}} - } - - def test_large_list_value_field_with_metadata(self): - vf = pa.field("item", pa.int32(), metadata={b"k": b"v"}) - field = pa.field("lst", pa.large_list(vf)) - result = _serialize_field_meta_tree(field) - assert result == { - "children": {"item": {"meta": {b64(b"k"): b64(b"v")}}} - } - - def test_fixed_size_list_value_field_with_metadata(self): - vf = pa.field("item", pa.float32(), metadata={b"k": b"v"}) - field = pa.field("lst", pa.list_(vf, 3)) - result = _serialize_field_meta_tree(field) - assert result == { - "children": {"item": {"meta": {b64(b"k"): b64(b"v")}}} - } - - def test_nested_struct_in_struct(self): - leaf = pa.field("z", pa.int8(), metadata={b"deep": b"yes"}) - inner_struct = pa.field("inner", pa.struct([leaf])) - outer = pa.field("outer", pa.struct([inner_struct])) - result = _serialize_field_meta_tree(outer) - assert result == { - "children": { - "inner": { - "children": {"z": {"meta": {b64(b"deep"): b64(b"yes")}}} - } - } - } - - def test_no_metadata_anywhere_returns_none(self): - inner = pa.field("a", pa.int64()) - field = pa.field("s", pa.struct([inner])) - assert _serialize_field_meta_tree(field) is None - - -# --------------------------------------------------------------------------- -# _serialize_arrow_metadata -# --------------------------------------------------------------------------- - - -class TestSerializeArrowMetadata: - def test_no_metadata_returns_none(self): - table = pa.table({"x": pa.array([1, 2], type=pa.int64())}) - assert _serialize_arrow_metadata(table) is None - - def test_schema_metadata_only(self): - table = pa.table( - {"x": pa.array([1], type=pa.int64())}, - schema=pa.schema( - [pa.field("x", pa.int64())], - metadata={b"origin": b"test"}, - ), - ) - result = _serialize_arrow_metadata(table) - assert result is not None - assert _ARROW_METADATA_KEY in result - blob = json.loads(result[_ARROW_METADATA_KEY].decode()) - assert blob["schema"] == {b64(b"origin"): b64(b"test")} - assert "fields" not in blob - - def test_field_metadata_only(self): - schema = pa.schema([pa.field("x", pa.int64(), metadata={b"unit": b"m"})]) - table = pa.table({"x": pa.array([1], type=pa.int64())}, schema=schema) - result = _serialize_arrow_metadata(table) - assert result is not None - blob = json.loads(result[_ARROW_METADATA_KEY].decode()) - assert "schema" not in blob - assert blob["fields"]["x"] == {"meta": {b64(b"unit"): b64(b"m")}} - - def test_nested_field_metadata(self): - inner = pa.field("val", pa.float64(), metadata={b"k": b"v"}) - schema = pa.schema([pa.field("s", pa.struct([inner]))]) - table = pa.table( - {"s": pa.array([{"val": 1.0}], type=pa.struct([pa.field("val", pa.float64())]))}, - schema=schema, - ) - result = _serialize_arrow_metadata(table) - assert result is not None - blob = json.loads(result[_ARROW_METADATA_KEY].decode()) - assert blob["fields"]["s"] == { - "children": {"val": {"meta": {b64(b"k"): b64(b"v")}}} - } - - def test_returns_bytes_value(self): - schema = pa.schema( - [pa.field("x", pa.int64())], metadata={b"k": b"v"} - ) - table = pa.table({"x": pa.array([1])}, schema=schema) - result = _serialize_arrow_metadata(table) - assert isinstance(result[_ARROW_METADATA_KEY], bytes) -``` - -- [ ] **Step 2: Run to verify all fail** - -```bash -uv run pytest tests/test_databases/test_spiraldb_metadata_helpers.py::TestSerializeFieldMetaTree tests/test_databases/test_spiraldb_metadata_helpers.py::TestSerializeArrowMetadata -v --tb=short -``` - -Expected: `ImportError` or `ModuleNotFoundError` — helpers don't exist yet. - -- [ ] **Step 3: Implement `_ARROW_METADATA_KEY`, `_serialize_field_meta_tree`, `_serialize_arrow_metadata`** - -Add immediately after the existing module-level `logger = logging.getLogger(__name__)` line in `src/orcapod/databases/spiraldb_connector.py`: - -```python -import base64 -import json - -_ARROW_METADATA_KEY = "__arrow_metadata__" - - -def _serialize_field_meta_tree(field: "pa.Field") -> "dict | None": - """Recursively build the metadata tree for a single ``pa.Field``. - - Walks the field's type tree, collecting ``field.metadata`` at each level. - Covers struct inner fields, list/large_list/fixed_size_list value fields. - Map key/item fields are not recursed (``pa.map_`` constructor does not - accept field-level metadata on key/item). - - Args: - field: The ``pa.Field`` to serialize. - - Returns: - A dict with optional ``"meta"`` (base64-encoded k/v pairs) and - ``"children"`` (recursive trees for composite type children) keys, - or ``None`` if this field and all its descendants have no metadata. - """ - import pyarrow as _pa # noqa: PLC0415 - - node: dict = {} - - if field.metadata: - node["meta"] = { - base64.b64encode(k).decode(): base64.b64encode(v).decode() - for k, v in field.metadata.items() - } - - ftype = field.type - child_fields: list = [] - if _pa.types.is_struct(ftype): - child_fields = [ftype.field(i) for i in range(ftype.num_fields)] - elif ( - _pa.types.is_list(ftype) - or _pa.types.is_large_list(ftype) - or _pa.types.is_fixed_size_list(ftype) - ): - child_fields = [ftype.value_field] - - if child_fields: - children: dict = {} - for child in child_fields: - child_tree = _serialize_field_meta_tree(child) - if child_tree is not None: - children[child.name] = child_tree - if children: - node["children"] = children - - return node if node else None - - -def _serialize_arrow_metadata(table: "pa.Table") -> "dict[str, bytes] | None": - """Encode all Arrow metadata from ``table`` into a single SpiralDB KV entry. - - Serializes both schema-level (``table.schema.metadata``) and per-field - metadata (including nested struct/list field metadata) into a single JSON - blob stored under ``_ARROW_METADATA_KEY``. - - Args: - table: The Arrow table whose metadata to encode. - - Returns: - ``{"__arrow_metadata__": blob_bytes}`` if any metadata exists anywhere - in the schema, or ``None`` if the schema and all fields (at every depth) - have no metadata. - """ - blob: dict = {} - - if table.schema.metadata: - blob["schema"] = { - base64.b64encode(k).decode(): base64.b64encode(v).decode() - for k, v in table.schema.metadata.items() - } - - field_trees: dict = {} - for field in table.schema: - tree = _serialize_field_meta_tree(field) - if tree is not None: - field_trees[field.name] = tree - if field_trees: - blob["fields"] = field_trees - - if not blob: - return None - - return {_ARROW_METADATA_KEY: json.dumps(blob).encode("utf-8")} -``` - -- [ ] **Step 4: Run serialization tests — verify they pass** - -```bash -uv run pytest tests/test_databases/test_spiraldb_metadata_helpers.py::TestSerializeFieldMetaTree tests/test_databases/test_spiraldb_metadata_helpers.py::TestSerializeArrowMetadata -v -``` - -Expected: all pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/databases/spiraldb_connector.py tests/test_databases/test_spiraldb_metadata_helpers.py -git commit -m "feat(spiraldb): add _serialize_field_meta_tree and _serialize_arrow_metadata helpers (ITL-471)" -``` - ---- - -## Task 3: Deserialization helpers — `_load_arrow_metadata` and `_restore_field` - -**Files:** -- Modify: `tests/test_databases/test_spiraldb_metadata_helpers.py` (append new test classes) -- Modify: `src/orcapod/databases/spiraldb_connector.py` (append new helpers) - -- [ ] **Step 1: Write failing tests for `_load_arrow_metadata` and `_restore_field`** - -Append to `tests/test_databases/test_spiraldb_metadata_helpers.py`: - -```python -# --------------------------------------------------------------------------- -# _load_arrow_metadata -# --------------------------------------------------------------------------- - - -class TestLoadArrowMetadata: - def test_empty_kv_returns_none_and_empty(self): - schema_meta, field_trees = _load_arrow_metadata({}) - assert schema_meta is None - assert field_trees == {} - - def test_missing_key_returns_none_and_empty(self): - schema_meta, field_trees = _load_arrow_metadata({"other_key": b"data"}) - assert schema_meta is None - assert field_trees == {} - - def test_schema_metadata_decoded(self): - blob = json.dumps({ - "schema": {b64(b"origin"): b64(b"test")} - }).encode() - schema_meta, field_trees = _load_arrow_metadata({_ARROW_METADATA_KEY: blob}) - assert schema_meta == {b"origin": b"test"} - assert field_trees == {} - - def test_field_trees_returned_raw(self): - blob = json.dumps({ - "fields": {"x": {"meta": {b64(b"unit"): b64(b"m")}}} - }).encode() - schema_meta, field_trees = _load_arrow_metadata({_ARROW_METADATA_KEY: blob}) - assert schema_meta is None - assert "x" in field_trees - assert field_trees["x"] == {"meta": {b64(b"unit"): b64(b"m")}} - - def test_roundtrip_with_serialize(self): - schema = pa.schema( - [pa.field("x", pa.int64(), metadata={b"unit": b"m"})], - metadata={b"origin": b"test"}, - ) - table = pa.table({"x": pa.array([1])}, schema=schema) - kv = _serialize_arrow_metadata(table) - schema_meta, field_trees = _load_arrow_metadata(kv) - assert schema_meta == {b"origin": b"test"} - assert "x" in field_trees - - -# --------------------------------------------------------------------------- -# _restore_field -# --------------------------------------------------------------------------- - - -class TestRestoreField: - def test_none_stored_returns_field_unchanged(self): - field = pa.field("x", pa.int64(), metadata={b"k": b"v"}) - restored, changed = _restore_field(field, None) - assert restored == field - assert not changed - - def test_restores_own_metadata(self): - field = pa.field("x", pa.int64()) - stored = {"meta": {b64(b"unit"): b64(b"m")}} - restored, changed = _restore_field(field, stored) - assert changed - assert restored.metadata == {b"unit": b"m"} - assert restored.type == pa.int64() - assert restored.name == "x" - - def test_same_metadata_not_changed(self): - field = pa.field("x", pa.int64(), metadata={b"k": b"v"}) - stored = {"meta": {b64(b"k"): b64(b"v")}} - restored, changed = _restore_field(field, stored) - # metadata is identical — no change needed - assert not changed - - def test_restores_struct_child_metadata(self): - inner = pa.field("val", pa.float64()) - field = pa.field("s", pa.struct([inner])) - stored = { - "children": {"val": {"meta": {b64(b"k"): b64(b"v")}}} - } - restored, changed = _restore_field(field, stored) - assert changed - assert pa.types.is_struct(restored.type) - restored_inner = restored.type.field("val") - assert restored_inner.metadata == {b"k": b"v"} - - def test_restores_list_value_field_metadata(self): - vf = pa.field("item", pa.int32()) - field = pa.field("lst", pa.list_(vf)) - stored = {"children": {"item": {"meta": {b64(b"k"): b64(b"v")}}}} - restored, changed = _restore_field(field, stored) - assert changed - assert pa.types.is_list(restored.type) - assert restored.type.value_field.metadata == {b"k": b"v"} - - def test_restores_large_list_value_field_metadata(self): - vf = pa.field("item", pa.int32()) - field = pa.field("lst", pa.large_list(vf)) - stored = {"children": {"item": {"meta": {b64(b"k"): b64(b"v")}}}} - restored, changed = _restore_field(field, stored) - assert changed - assert pa.types.is_large_list(restored.type) - assert restored.type.value_field.metadata == {b"k": b"v"} - - def test_no_children_stored_struct_unchanged(self): - inner = pa.field("val", pa.float64()) - field = pa.field("s", pa.struct([inner])) - # stored has meta for the outer field but no children - stored = {"meta": {b64(b"tag"): b64(b"yes")}} - restored, changed = _restore_field(field, stored) - assert changed - assert restored.metadata == {b"tag": b"yes"} - # inner field should be unchanged - assert restored.type.field("val").metadata is None - - def test_nested_struct_in_struct_restored(self): - leaf = pa.field("z", pa.int8()) - inner_struct = pa.field("inner", pa.struct([leaf])) - outer = pa.field("outer", pa.struct([inner_struct])) - stored = { - "children": { - "inner": { - "children": {"z": {"meta": {b64(b"deep"): b64(b"yes")}}} - } - } - } - restored, changed = _restore_field(outer, stored) - assert changed - inner_type = restored.type.field("inner").type - assert inner_type.field("z").metadata == {b"deep": b"yes"} - - def test_full_roundtrip_serialize_then_restore(self): - """Serialize a field's metadata tree then restore it — must be identity.""" - vf = pa.field("item", pa.int32(), metadata={b"unit": b"bytes"}) - field = pa.field("lst", pa.list_(vf), metadata={b"role": b"data"}) - - tree = _serialize_field_meta_tree(field) - restored, changed = _restore_field(field, tree) - assert changed is False # already has this metadata, nothing changed - assert restored.metadata == field.metadata - assert restored.type.value_field.metadata == vf.metadata -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_databases/test_spiraldb_metadata_helpers.py::TestLoadArrowMetadata tests/test_databases/test_spiraldb_metadata_helpers.py::TestRestoreField -v --tb=short -``` - -Expected: `ImportError` — `_load_arrow_metadata` and `_restore_field` not defined yet. - -- [ ] **Step 3: Implement `_load_arrow_metadata`, `_restore_arrow_type`, and `_restore_field`** - -Append to `src/orcapod/databases/spiraldb_connector.py` (after `_serialize_arrow_metadata`): - -```python -def _load_arrow_metadata( - kv: "dict[str, bytes]", -) -> "tuple[dict[bytes, bytes] | None, dict[str, dict]]": - """Decode Arrow metadata from a SpiralDB table KV store. - - Args: - kv: The raw KV dict from ``Table.get_metadata()``. - - Returns: - A tuple ``(schema_meta, field_trees)`` where ``schema_meta`` is None - if absent and ``field_trees`` maps top-level column name to its raw - metadata tree dict (as stored in the blob by ``_serialize_arrow_metadata``). - """ - raw = kv.get(_ARROW_METADATA_KEY) - if raw is None: - return None, {} - - blob = json.loads(raw.decode("utf-8")) - - schema_meta: "dict[bytes, bytes] | None" = None - if "schema" in blob: - schema_meta = { - base64.b64decode(k): base64.b64decode(v) - for k, v in blob["schema"].items() - } - - field_trees: "dict[str, dict]" = blob.get("fields", {}) - return schema_meta, field_trees - - -def _restore_arrow_type( - arrow_type: "pa.DataType", - children_stored: "dict[str, dict]", -) -> "tuple[pa.DataType, bool]": - """Recursively restore nested field metadata within an Arrow type. - - Covers ``pa.struct`` (all inner fields), ``pa.list_`` / ``pa.large_list`` - / ``pa.fixed_size_list`` (value field). ``pa.map_`` key/item fields are - not recursed — the ``pa.map_`` constructor does not accept field-level - metadata on key or item. - - Args: - arrow_type: The Arrow type whose nested fields to restore. - children_stored: Dict of child field name → raw metadata tree. - - Returns: - ``(new_type, changed)`` — ``changed`` is True if any descendant was - modified. - """ - import pyarrow as _pa # noqa: PLC0415 - - if _pa.types.is_struct(arrow_type): - new_fields = [] - any_changed = False - for i in range(arrow_type.num_fields): - child = arrow_type.field(i) - restored_child, changed = _restore_field(child, children_stored.get(child.name)) - new_fields.append(restored_child) - any_changed = any_changed or changed - if any_changed: - return _pa.struct(new_fields), True - return arrow_type, False - - if _pa.types.is_list(arrow_type): - vf = arrow_type.value_field - restored_vf, changed = _restore_field(vf, children_stored.get(vf.name)) - if changed: - return _pa.list_(restored_vf), True - return arrow_type, False - - if _pa.types.is_large_list(arrow_type): - vf = arrow_type.value_field - restored_vf, changed = _restore_field(vf, children_stored.get(vf.name)) - if changed: - return _pa.large_list(restored_vf), True - return arrow_type, False - - if _pa.types.is_fixed_size_list(arrow_type): - vf = arrow_type.value_field - restored_vf, changed = _restore_field(vf, children_stored.get(vf.name)) - if changed: - return _pa.list_(restored_vf, arrow_type.list_size), True - return arrow_type, False - - return arrow_type, False - - -def _restore_field( - field: "pa.Field", - stored: "dict | None", -) -> "tuple[pa.Field, bool]": - """Recursively restore a field's metadata and rebuild its nested type. - - Walks the stored metadata tree in parallel with the field's type tree, - reattaching ``field.metadata`` at every level and reconstructing composite - types bottom-up when any descendant has metadata to restore. - - Args: - field: The ``pa.Field`` from the wire (SpiralDB scan output). - stored: The raw metadata tree dict for this field, as decoded by - ``_load_arrow_metadata``. ``None`` if no stored metadata exists - for this field. - - Returns: - ``(restored_field, changed)`` where ``changed`` is True if any - metadata or type was modified relative to ``field``. - """ - import base64 as _base64 # noqa: PLC0415 - import pyarrow as _pa # noqa: PLC0415 - - if stored is None: - return field, False - - # Decode this field's own metadata - restored_meta: "dict[bytes, bytes] | None" = None - if "meta" in stored: - restored_meta = { - _base64.b64decode(k): _base64.b64decode(v) - for k, v in stored["meta"].items() - } - - # Recursively restore the nested type - children_stored = stored.get("children", {}) - new_type, type_changed = _restore_arrow_type(field.type, children_stored) - - meta_changed = restored_meta is not None and restored_meta != field.metadata - if not meta_changed and not type_changed: - return field, False - - final_meta = restored_meta if restored_meta is not None else field.metadata - return _pa.field(field.name, new_type, field.nullable, final_meta), True -``` - -- [ ] **Step 4: Run deserialization tests — verify they pass** - -```bash -uv run pytest tests/test_databases/test_spiraldb_metadata_helpers.py -v -``` - -Expected: all pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/databases/spiraldb_connector.py tests/test_databases/test_spiraldb_metadata_helpers.py -git commit -m "feat(spiraldb): add _load_arrow_metadata and _restore_field helpers (ITL-471)" -``` - ---- - -## Task 4: `validate_records` — protocol, connectors, and `ConnectorArrowDatabase` - -**Files:** -- Modify: `src/orcapod/protocols/db_connector_protocol.py` -- Modify: `src/orcapod/databases/sqlite_connector.py` -- Modify: `src/orcapod/databases/postgresql_connector.py` -- Modify: `src/orcapod/databases/spiraldb_connector.py` -- Modify: `src/orcapod/databases/connector_arrow_database.py` -- Modify: `tests/test_databases/test_connector_arrow_database.py` -- Modify: `tests/test_databases/test_spiraldb_connector.py` - -- [ ] **Step 1: Write failing tests** - -**In `tests/test_databases/test_spiraldb_connector.py`**, append a new test class: - -```python -class TestValidateRecords: - def test_no_op_for_plain_columns(self, connector): - """validate_records is a no-op on SpiralDBConnector for plain columns.""" - table = pa.table({ - "id": pa.array([b"a"], type=pa.large_binary()), - "value": pa.array([1], type=pa.int64()), - }) - connector.validate_records(table) # must not raise - - def test_no_op_for_extension_typed_columns(self, connector): - """SpiralDBConnector.validate_records allows extension-typed columns.""" - ext_field = pa.field( - "payload", - pa.large_string(), - metadata={ - b"ARROW:extension:name": b"orcapod.path", - b"ARROW:extension:metadata": b"", - }, - ) - schema = pa.schema([pa.field("id", pa.large_binary()), ext_field]) - table = pa.table( - {"id": pa.array([b"a"], type=pa.large_binary()), - "payload": pa.array(["/tmp/x"], type=pa.large_string())}, - schema=schema, - ) - connector.validate_records(table) # must not raise — SpiralDB supports extension types -``` - -**In `tests/test_databases/test_connector_arrow_database.py`**, update `TestExtensionTypeWriteGuard`: - -The fixture uses `MockDBConnector()`. `MockDBConnector` must gain a `validate_records` method. The existing tests (`test_rejects_in_memory_extension_type_column`, `test_rejects_metadata_only_extension_column`) should still raise `ValueError` because `MockDBConnector` uses the rejecting default. `test_plain_column_not_rejected` should still pass. - -Add a new test at the end of `TestExtensionTypeWriteGuard`: - -```python - def test_permissive_connector_allows_extension_types(self): - """A connector with a permissive validate_records lets extension columns through.""" - import pyarrow as pa - - class PermissiveConnector(MockDBConnector): - def validate_records(self, records): - pass # no-op — this connector supports metadata - - db = ConnectorArrowDatabase(PermissiveConnector()) - ext_field = pa.field( - "payload", - pa.large_string(), - metadata={ - b"ARROW:extension:name": b"orcapod.path", - b"ARROW:extension:metadata": b"", - }, - ) - schema = pa.schema([pa.field("__record_id", pa.large_binary()), ext_field]) - table = pa.table( - {"__record_id": pa.array([b"id1"], type=pa.large_binary()), - "payload": pa.array(["/tmp/test"], type=pa.large_string())}, - schema=schema, - ) - db.add_records(("results",), table, record_id_column="__record_id") # must not raise -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py::TestValidateRecords tests/test_databases/test_connector_arrow_database.py::TestExtensionTypeWriteGuard::test_permissive_connector_allows_extension_types -v --tb=short -``` - -Expected: `AttributeError: 'SpiralDBConnector' object has no attribute 'validate_records'` - -- [ ] **Step 3: Add `validate_records` to `DBConnectorProtocol`** - -In `src/orcapod/protocols/db_connector_protocol.py`, add after the `upsert_records` method (before the `# ── Lifecycle` section): - -```python - def validate_records(self, records: "pa.Table") -> None: - """Validate that ``records`` are safe to write through this connector. - - Implementations should raise ``ValueError`` if ``records`` contain - columns that this connector cannot round-trip faithfully (e.g. Arrow - extension-typed columns on connectors that drop field metadata). - - A connector that fully preserves Arrow field metadata should implement - this as a no-op. - - Args: - records: Arrow table to validate before writing. - - Raises: - ValueError: If any column is incompatible with this connector. - """ - ... -``` - -- [ ] **Step 4: Add rejecting `validate_records` to `SQLiteConnector`** - -In `src/orcapod/databases/sqlite_connector.py`, add after the `upsert_records` method (before `def close`): - -```python - def validate_records(self, records: "pa.Table") -> None: - """Reject Arrow extension-typed columns. - - ``SQLiteConnector`` does not preserve ``ARROW:extension:*`` field - metadata — extension types would be silently demoted to their storage - type on read. - - Args: - records: Arrow table to validate. - - Raises: - ValueError: If any column carries an Arrow extension type. - """ - import pyarrow as _pa # noqa: PLC0415 - - _EXT_NAME_KEY = b"ARROW:extension:name" - ext_fields: list[tuple[str, str]] = [] - for field in records.schema: - if isinstance(field.type, _pa.ExtensionType): - ext_fields.append((field.name, field.type.extension_name)) - elif field.metadata and _EXT_NAME_KEY in field.metadata: - ext_fields.append( - (field.name, field.metadata[_EXT_NAME_KEY].decode("utf-8", errors="replace")) - ) - if ext_fields: - ext_info = ", ".join(f"{name!r}: {ext_name!r}" for name, ext_name in ext_fields) - raise ValueError( - f"SQLiteConnector does not support Arrow extension-typed columns " - f"({ext_info}). SQLiteConnector does not preserve ARROW:extension:* " - "field metadata across read/write cycles." - ) -``` - -- [ ] **Step 5: Add rejecting `validate_records` to `PostgreSQLConnector`** - -In `src/orcapod/databases/postgresql_connector.py`, add after the `upsert_records` method (before `def close`): - -```python - def validate_records(self, records: "pa.Table") -> None: - """Reject Arrow extension-typed columns. - - ``PostgreSQLConnector`` does not preserve ``ARROW:extension:*`` field - metadata — extension types would be silently demoted to their storage - type on read. - - Args: - records: Arrow table to validate. - - Raises: - ValueError: If any column carries an Arrow extension type. - """ - import pyarrow as _pa # noqa: PLC0415 - - _EXT_NAME_KEY = b"ARROW:extension:name" - ext_fields: list[tuple[str, str]] = [] - for field in records.schema: - if isinstance(field.type, _pa.ExtensionType): - ext_fields.append((field.name, field.type.extension_name)) - elif field.metadata and _EXT_NAME_KEY in field.metadata: - ext_fields.append( - (field.name, field.metadata[_EXT_NAME_KEY].decode("utf-8", errors="replace")) - ) - if ext_fields: - ext_info = ", ".join(f"{name!r}: {ext_name!r}" for name, ext_name in ext_fields) - raise ValueError( - f"PostgreSQLConnector does not support Arrow extension-typed columns " - f"({ext_info}). PostgreSQLConnector does not preserve ARROW:extension:* " - "field metadata across read/write cycles." - ) -``` - -- [ ] **Step 6: Add no-op `validate_records` to `SpiralDBConnector`** - -In `src/orcapod/databases/spiraldb_connector.py`, add after the `upsert_records` method (before `def close`): - -```python - def validate_records(self, records: "pa.Table") -> None: - """No-op: ``SpiralDBConnector`` preserves Arrow field metadata via the native KV store.""" -``` - -- [ ] **Step 7: Add `validate_records` to `MockDBConnector` in the test file** - -In `tests/test_databases/test_connector_arrow_database.py`, add after `MockDBConnector.upsert_records` (before `def close`): - -```python - def validate_records(self, records: "pa.Table") -> None: - """Rejecting default — mirrors real SQL connectors for test fidelity.""" - import pyarrow as _pa # noqa: PLC0415 - - _EXT_NAME_KEY = b"ARROW:extension:name" - ext_fields: list[tuple[str, str]] = [] - for field in records.schema: - if isinstance(field.type, _pa.ExtensionType): - ext_fields.append((field.name, field.type.extension_name)) - elif field.metadata and _EXT_NAME_KEY in field.metadata: - ext_fields.append( - (field.name, field.metadata[_EXT_NAME_KEY].decode("utf-8", errors="replace")) - ) - if ext_fields: - ext_info = ", ".join(f"{name!r}: {ext_name!r}" for name, ext_name in ext_fields) - raise ValueError( - f"MockDBConnector does not support Arrow extension-typed columns ({ext_info})." - ) -``` - -- [ ] **Step 8: Replace the inline guard in `ConnectorArrowDatabase.add_records`** - -In `src/orcapod/databases/connector_arrow_database.py`, find the block starting with: - -```python - # Reject Arrow extension-typed columns: SQL connectors do not preserve - # ARROW:extension:* field metadata, so extension types would be silently - # dropped on read, making round-trips impossible. Use DeltaTableDatabase - # or write directly to Parquet instead. See PLT-1795 for the planned fix. - # - # Two representations are checked: - # 1. In-memory extension types: isinstance(field.type, pa.ExtensionType). - # 2. Metadata-only extension columns: a plain Arrow type whose field metadata - # contains the b"ARROW:extension:name" key. This arises when reading a - # Parquet/IPC file with an unregistered extension type — the array is - # decoded as its storage type but the metadata is preserved on the field. - _EXT_NAME_KEY = b"ARROW:extension:name" - ext_fields: list[tuple[str, str]] = [] - for field in records.schema: - if isinstance(field.type, pa.ExtensionType): - ext_fields.append((field.name, field.type.extension_name)) - elif field.metadata and _EXT_NAME_KEY in field.metadata: - ext_fields.append((field.name, field.metadata[_EXT_NAME_KEY].decode("utf-8", errors="replace"))) - if ext_fields: - ext_info = ", ".join(f"{name!r}: {ext_name!r}" for name, ext_name in ext_fields) - raise ValueError( - f"ConnectorArrowDatabase does not support Arrow extension-typed columns " - f"({ext_info}). SQL connectors do not preserve ARROW:extension:* field " - f"metadata, so extension types would be silently dropped on read. " - f"Use DeltaTableDatabase or write directly to Parquet instead. " - f"See PLT-1795 for the planned fix." - ) -``` - -Replace it entirely with: - -```python - self._connector.validate_records(records) -``` - -- [ ] **Step 9: Run validate_records tests** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py::TestValidateRecords tests/test_databases/test_connector_arrow_database.py::TestExtensionTypeWriteGuard -v -``` - -Expected: all pass. - -- [ ] **Step 10: Run full test suite to check nothing broke** - -```bash -uv run pytest tests/test_databases/ -v --tb=short -q -``` - -Expected: all pass (integration tests skipped). - -- [ ] **Step 11: Commit** - -```bash -git add \ - src/orcapod/protocols/db_connector_protocol.py \ - src/orcapod/databases/sqlite_connector.py \ - src/orcapod/databases/postgresql_connector.py \ - src/orcapod/databases/spiraldb_connector.py \ - src/orcapod/databases/connector_arrow_database.py \ - tests/test_databases/test_connector_arrow_database.py \ - tests/test_databases/test_spiraldb_connector.py -git commit -m "feat(spiraldb): add per-connector validate_records hook, replace ConnectorArrowDatabase guard (ITL-471)" -``` - ---- - -## Task 5: Write path — persist metadata in `upsert_records` - -**Files:** -- Modify: `tests/test_databases/test_spiraldb_connector.py` (append new test class) -- Modify: `src/orcapod/databases/spiraldb_connector.py` - -- [ ] **Step 1: Write failing tests** - -Append to `tests/test_databases/test_spiraldb_connector.py`: - -```python -class TestUpsertRecordsMetadata: - def _make_mock_table(self, mock_project, pk_cols: list[str]) -> MagicMock: - mock_table = MagicMock() - mock_table.key_schema.names = pk_cols - mock_project.table.return_value = mock_table - return mock_table - - def test_set_metadata_called_after_write_when_schema_metadata_present( - self, connector, mock_project - ): - mock_tbl = self._make_mock_table(mock_project, ["id"]) - schema = pa.schema( - [pa.field("id", pa.string()), pa.field("val", pa.int64())], - metadata={b"origin": b"test"}, - ) - records = pa.table( - {"id": pa.array(["a"]), "val": pa.array([1], type=pa.int64())}, - schema=schema, - ) - connector.upsert_records("t", records, id_column="id") - mock_tbl.write.assert_called_once() - mock_tbl.set_metadata.assert_called_once() - kv = mock_tbl.set_metadata.call_args[0][0] - assert "__arrow_metadata__" in kv - - def test_set_metadata_called_when_field_metadata_present( - self, connector, mock_project - ): - mock_tbl = self._make_mock_table(mock_project, ["id"]) - schema = pa.schema([ - pa.field("id", pa.string()), - pa.field("val", pa.int64(), metadata={b"unit": b"m"}), - ]) - records = pa.table( - {"id": pa.array(["a"]), "val": pa.array([1], type=pa.int64())}, - schema=schema, - ) - connector.upsert_records("t", records, id_column="id") - mock_tbl.set_metadata.assert_called_once() - - def test_set_metadata_not_called_when_no_metadata( - self, connector, mock_project - ): - mock_tbl = self._make_mock_table(mock_project, ["id"]) - records = pa.table({"id": pa.array(["a"]), "val": pa.array([1])}) - connector.upsert_records("t", records, id_column="id") - mock_tbl.write.assert_called_once() - mock_tbl.set_metadata.assert_not_called() - - def test_set_metadata_called_for_skip_existing_path( - self, connector, mock_sp, mock_project - ): - mock_tbl = self._make_mock_table(mock_project, ["id"]) - existing = pa.table({"id": pa.array(["a"]), "val": pa.array([1])}) - mock_sp.Spiral.return_value.scan.return_value.to_table.return_value = existing - schema = pa.schema( - [pa.field("id", pa.string()), pa.field("val", pa.int64())], - metadata={b"version": b"1"}, - ) - records = pa.table( - {"id": pa.array(["a", "b"]), "val": pa.array([1, 2], type=pa.int64())}, - schema=schema, - ) - connector.upsert_records("t", records, id_column="id", skip_existing=True) - # "b" is novel — write called; metadata from original records, not filtered subset - mock_tbl.write.assert_called_once() - mock_tbl.set_metadata.assert_called_once() - kv = mock_tbl.set_metadata.call_args[0][0] - import json, base64 - blob = json.loads(kv["__arrow_metadata__"].decode()) - assert blob["schema"][base64.b64encode(b"version").decode()] == base64.b64encode(b"1").decode() - - def test_set_metadata_not_called_when_skip_existing_all_exist( - self, connector, mock_sp, mock_project - ): - mock_tbl = self._make_mock_table(mock_project, ["id"]) - existing = pa.table({"id": pa.array(["a", "b"]), "val": pa.array([1, 2])}) - mock_sp.Spiral.return_value.scan.return_value.to_table.return_value = existing - records = pa.table({"id": pa.array(["a", "b"]), "val": pa.array([10, 20])}) - connector.upsert_records("t", records, id_column="id", skip_existing=True) - mock_tbl.write.assert_not_called() - mock_tbl.set_metadata.assert_not_called() -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py::TestUpsertRecordsMetadata -v --tb=short -``` - -Expected: `AssertionError` — `set_metadata` not called (feature not implemented). - -- [ ] **Step 3: Modify `upsert_records` to write metadata** - -In `src/orcapod/databases/spiraldb_connector.py`, find the `upsert_records` method. The current structure is: - -```python - if not skip_existing: - # Always upsert-by-key: existing rows overwritten, novel rows inserted. - tbl.write(records) - return - - # skip_existing=True: full scan → client-side key filter → write novel rows. - existing = self._spiral.scan(tbl.select()).to_table() - ... - if len(novel) > 0: - tbl.write(novel) -``` - -Replace the body after the guards (starting from `if not skip_existing:`) with: - -```python - meta_kv = _serialize_arrow_metadata(records) - - if not skip_existing: - # Always upsert-by-key: existing rows overwritten, novel rows inserted. - tbl.write(records) - if meta_kv is not None: - tbl.set_metadata(meta_kv) - return - - # skip_existing=True: full scan → client-side key filter → write novel rows. - existing = self._spiral.scan(tbl.select()).to_table() - existing_keys = { - tuple(row[k] for k in pk_cols) - for row in existing.to_pylist() - } - mask = pa.array( - [ - tuple(row[k] for k in pk_cols) not in existing_keys - for row in records.to_pylist() - ], - type=pa.bool_(), - ) - novel = records.filter(mask) - if len(novel) > 0: - tbl.write(novel) - if meta_kv is not None: - tbl.set_metadata(meta_kv) -``` - -- [ ] **Step 4: Run write-path tests** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py::TestUpsertRecordsMetadata -v -``` - -Expected: all pass. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/databases/spiraldb_connector.py tests/test_databases/test_spiraldb_connector.py -git commit -m "feat(spiraldb): persist Arrow metadata in upsert_records via KV store (ITL-471)" -``` - ---- - -## Task 6: Read path — restore metadata in `iter_batches` - -**Files:** -- Modify: `tests/test_databases/test_spiraldb_connector.py` (append new test class) -- Modify: `src/orcapod/databases/spiraldb_connector.py` - -- [ ] **Step 1: Write failing tests** - -Append to `tests/test_databases/test_spiraldb_connector.py`: - -```python -class TestIterBatchesMetadata: - def _wire_scan(self, mock_sp, mock_project, batches: list, kv: dict | None = None) -> None: - mock_project.table.return_value.select.return_value = MagicMock() - mock_project.table.return_value.get_metadata.return_value = kv or {} - mock_sp.Spiral.return_value.scan.return_value.to_record_batches.return_value = iter(batches) - - def test_schema_metadata_reattached(self, connector, mock_sp, mock_project): - import json, base64 - stored_blob = json.dumps({ - "schema": { - base64.b64encode(b"origin").decode(): base64.b64encode(b"test").decode() - } - }).encode() - raw_batch = pa.record_batch({"id": pa.array(["a"], type=pa.string())}) - self._wire_scan(mock_sp, mock_project, [raw_batch], {"__arrow_metadata__": stored_blob}) - - result = list(connector.iter_batches('SELECT * FROM "t"')) - assert len(result) == 1 - assert result[0].schema.metadata == {b"origin": b"test"} - - def test_field_metadata_reattached(self, connector, mock_sp, mock_project): - import json, base64 - stored_blob = json.dumps({ - "fields": { - "val": {"meta": { - base64.b64encode(b"unit").decode(): base64.b64encode(b"meters").decode() - }} - } - }).encode() - raw_batch = pa.record_batch({ - "id": pa.array(["a"], type=pa.string()), - "val": pa.array([1.0], type=pa.float64()), - }) - self._wire_scan(mock_sp, mock_project, [raw_batch], {"__arrow_metadata__": stored_blob}) - - result = list(connector.iter_batches('SELECT * FROM "t"')) - assert result[0].schema.field("val").metadata == {b"unit": b"meters"} - - def test_string_normalization_preserved_with_metadata(self, connector, mock_sp, mock_project): - """string → large_string normalization still applies when field metadata present.""" - import json, base64 - stored_blob = json.dumps({ - "fields": { - "label": {"meta": { - base64.b64encode(b"ARROW:extension:name").decode(): base64.b64encode(b"orcapod.path").decode() - }} - } - }).encode() - # SpiralDB returns pa.string() at the wire - raw_batch = pa.record_batch({"label": pa.array(["x"], type=pa.string())}) - self._wire_scan(mock_sp, mock_project, [raw_batch], {"__arrow_metadata__": stored_blob}) - - result = list(connector.iter_batches('SELECT * FROM "t"')) - batch = result[0] - # type must be large_string (normalized) - assert batch.schema.field("label").type == pa.large_string() - # metadata must be restored - assert batch.schema.field("label").metadata == {b"ARROW:extension:name": b"orcapod.path"} - - def test_no_stored_metadata_behaves_as_before(self, connector, mock_sp, mock_project): - """Tables with no stored metadata are returned unchanged (backward compat).""" - raw_batch = pa.record_batch({"id": pa.array(["a"], type=pa.string()), "v": pa.array([1])}) - self._wire_scan(mock_sp, mock_project, [raw_batch], {}) - - result = list(connector.iter_batches('SELECT * FROM "t"')) - assert result[0].schema.field("id").type == pa.large_string() - assert result[0].schema.metadata is None - - def test_get_metadata_called_once_per_scan_not_per_batch(self, connector, mock_sp, mock_project): - b1 = pa.record_batch({"id": pa.array(["a"])}) - b2 = pa.record_batch({"id": pa.array(["b"])}) - self._wire_scan(mock_sp, mock_project, [b1, b2], {}) - list(connector.iter_batches('SELECT * FROM "t"')) - mock_project.table.return_value.get_metadata.assert_called_once() -``` - -- [ ] **Step 2: Run to verify they fail** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py::TestIterBatchesMetadata -v --tb=short -``` - -Expected: `AssertionError` — metadata not reattached (feature not implemented). - -- [ ] **Step 3: Modify `iter_batches` to restore metadata** - -In `src/orcapod/databases/spiraldb_connector.py`, find `iter_batches`. Replace the body starting from `table_name = _parse_table_name(query)` onward: - -```python - table_name = _parse_table_name(query) - tbl = self._project.table(self._table_id(table_name)) - schema_meta, field_trees = _load_arrow_metadata(tbl.get_metadata()) - reader = self._spiral.scan(tbl.select()).to_record_batches() - for batch in reader: - schema = batch.schema - new_fields = [] - needs_rebuild = False - for field in schema: - # Recursively restore field metadata (including nested struct/list). - restored_field, field_changed = _restore_field(field, field_trees.get(field.name)) - # Apply large_string / large_binary normalization on the top-level type. - if restored_field.type == _pa.string(): - restored_field = _pa.field( - restored_field.name, - _pa.large_string(), - restored_field.nullable, - restored_field.metadata, - ) - field_changed = True - elif restored_field.type == _pa.binary(): - restored_field = _pa.field( - restored_field.name, - _pa.large_binary(), - restored_field.nullable, - restored_field.metadata, - ) - field_changed = True - new_fields.append(restored_field) - needs_rebuild = needs_rebuild or field_changed - - restored_schema_meta = schema_meta if schema_meta is not None else schema.metadata - if restored_schema_meta != schema.metadata: - needs_rebuild = True - - if needs_rebuild: - target_schema = _pa.schema(new_fields, metadata=restored_schema_meta) - batch = batch.cast(target_schema) - yield batch -``` - -- [ ] **Step 4: Run read-path tests** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector.py::TestIterBatchesMetadata -v -``` - -Expected: all pass. - -- [ ] **Step 5: Run full test suite** - -```bash -uv run pytest tests/test_databases/ -v --tb=short -q -``` - -Expected: all pass (integration tests skipped). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/databases/spiraldb_connector.py tests/test_databases/test_spiraldb_connector.py -git commit -m "feat(spiraldb): restore Arrow metadata in iter_batches via recursive _restore_field (ITL-471)" -``` - ---- - -## Task 7: Integration tests - -**Files:** -- Modify: `tests/test_databases/test_spiraldb_connector_integration.py` - -- [ ] **Step 1: Append `TestArrowMetadataRoundTrip` class** - -Append to `tests/test_databases/test_spiraldb_connector_integration.py`: - -```python -class TestArrowMetadataRoundTrip: - """Integration tests for Arrow metadata preservation across write→read cycles. - - Requires SPIRAL_INTEGRATION_TESTS=1 and valid SpiralDB credentials. - """ - - def test_schema_metadata_round_trip(self, connector): - """Schema-level metadata survives a full write→read cycle.""" - table_name = _unique_table("schema_meta") - schema = pa.schema( - [ - pa.field("__record_id", pa.string(), nullable=False), - pa.field("value", pa.float64()), - ], - metadata={b"origin": b"test-suite", b"version": b"1"}, - ) - records = pa.table( - {"__record_id": pa.array(["r1"]), "value": pa.array([1.0])}, - schema=schema, - ) - try: - connector.create_table_if_not_exists( - table_name, - columns=[ - ColumnInfo("__record_id", pa.string(), nullable=False), - ColumnInfo("value", pa.float64()), - ], - pk_column="__record_id", - ) - connector.upsert_records(table_name, records, id_column="__record_id") - - batches = list(connector.iter_batches(f'SELECT * FROM "{table_name}"')) - result = pa.Table.from_batches(batches) - assert result.schema.metadata is not None - assert result.schema.metadata.get(b"origin") == b"test-suite" - assert result.schema.metadata.get(b"version") == b"1" - finally: - connector.delete_table(table_name) - - def test_field_metadata_round_trip(self, connector): - """Per-column field metadata survives a full write→read cycle.""" - table_name = _unique_table("field_meta") - schema = pa.schema([ - pa.field("__record_id", pa.string(), nullable=False), - pa.field("value", pa.float64(), metadata={b"unit": b"meters", b"sensor": b"lidar"}), - ]) - records = pa.table( - {"__record_id": pa.array(["r1"]), "value": pa.array([42.0])}, - schema=schema, - ) - try: - connector.create_table_if_not_exists( - table_name, - columns=[ - ColumnInfo("__record_id", pa.string(), nullable=False), - ColumnInfo("value", pa.float64()), - ], - pk_column="__record_id", - ) - connector.upsert_records(table_name, records, id_column="__record_id") - - batches = list(connector.iter_batches(f'SELECT * FROM "{table_name}"')) - result = pa.Table.from_batches(batches) - value_meta = result.schema.field("value").metadata - assert value_meta is not None - assert value_meta.get(b"unit") == b"meters" - assert value_meta.get(b"sensor") == b"lidar" - finally: - connector.delete_table(table_name) - - def test_extension_type_metadata_round_trip(self, connector): - """ARROW:extension:name / ARROW:extension:metadata field metadata survives.""" - table_name = _unique_table("ext_meta") - ext_field = pa.field( - "path_col", - pa.large_string(), - metadata={ - b"ARROW:extension:name": b"orcapod.path", - b"ARROW:extension:metadata": b"", - }, - ) - schema = pa.schema([ - pa.field("__record_id", pa.string(), nullable=False), - ext_field, - ]) - records = pa.table( - { - "__record_id": pa.array(["r1"]), - "path_col": pa.array(["/tmp/test"], type=pa.large_string()), - }, - schema=schema, - ) - try: - connector.create_table_if_not_exists( - table_name, - columns=[ - ColumnInfo("__record_id", pa.string(), nullable=False), - ColumnInfo("path_col", pa.large_string()), - ], - pk_column="__record_id", - ) - connector.upsert_records(table_name, records, id_column="__record_id") - - batches = list(connector.iter_batches(f'SELECT * FROM "{table_name}"')) - result = pa.Table.from_batches(batches) - meta = result.schema.field("path_col").metadata - assert meta is not None - assert meta.get(b"ARROW:extension:name") == b"orcapod.path" - finally: - connector.delete_table(table_name) - - def test_no_metadata_backward_compatible(self, connector): - """Plain tables with no Arrow metadata read back without spurious KV entries.""" - table_name = _unique_table("no_meta") - try: - connector.create_table_if_not_exists( - table_name, - columns=[ - ColumnInfo("__record_id", pa.string(), nullable=False), - ColumnInfo("value", pa.int64()), - ], - pk_column="__record_id", - ) - records = pa.table({ - "__record_id": pa.array(["r1"]), - "value": pa.array([99], type=pa.int64()), - }) - connector.upsert_records(table_name, records, id_column="__record_id") - - batches = list(connector.iter_batches(f'SELECT * FROM "{table_name}"')) - result = pa.Table.from_batches(batches) - assert result.schema.metadata is None - for field in result.schema: - assert field.metadata is None - finally: - connector.delete_table(table_name) - - def test_connector_arrow_database_extension_type_round_trip(self, connector): - """Full ConnectorArrowDatabase path: write extension-typed column, read back intact.""" - unique_suffix = uuid.uuid4().hex[:8] - path = ("spiraldb", f"ext_roundtrip_{unique_suffix}") - table_name = f"spiraldb__ext_roundtrip_{unique_suffix}" - - ext_field = pa.field( - "path_col", - pa.large_string(), - metadata={ - b"ARROW:extension:name": b"orcapod.path", - b"ARROW:extension:metadata": b"", - }, - ) - schema = pa.schema([pa.field("__record_id", pa.large_binary()), ext_field]) - record = pa.table( - { - "__record_id": pa.array([b"test_r1"], type=pa.large_binary()), - "path_col": pa.array(["/data/file.npy"], type=pa.large_string()), - }, - schema=schema, - ) - - db = ConnectorArrowDatabase(connector) - try: - db.add_record(path, record_id=b"test_r1", record=record, flush=True) - result = db.get_record_by_id(path, b"test_r1") - assert result is not None - meta = result.schema.field("path_col").metadata - assert meta is not None - assert meta.get(b"ARROW:extension:name") == b"orcapod.path" - finally: - connector.delete_table(table_name) - - def test_nested_struct_field_metadata_round_trip(self, connector): - """Struct columns with inner-field metadata survive a full write→read cycle.""" - table_name = _unique_table("struct_meta") - inner_field = pa.field("val", pa.float64(), metadata={b"unit": b"volts"}) - struct_col = pa.field("measurement", pa.struct([inner_field])) - schema = pa.schema([ - pa.field("__record_id", pa.string(), nullable=False), - struct_col, - ]) - records = pa.table( - { - "__record_id": pa.array(["r1"]), - "measurement": pa.array([{"val": 3.14}], type=pa.struct([pa.field("val", pa.float64())])), - }, - schema=schema, - ) - try: - connector.create_table_if_not_exists( - table_name, - columns=[ - ColumnInfo("__record_id", pa.string(), nullable=False), - ColumnInfo("measurement", pa.struct([pa.field("val", pa.float64())])), - ], - pk_column="__record_id", - ) - connector.upsert_records(table_name, records, id_column="__record_id") - - batches = list(connector.iter_batches(f'SELECT * FROM "{table_name}"')) - result = pa.Table.from_batches(batches) - inner = result.schema.field("measurement").type.field("val") - assert inner.metadata is not None - assert inner.metadata.get(b"unit") == b"volts" - finally: - connector.delete_table(table_name) -``` - -- [ ] **Step 2: Verify integration tests are skipped (no credentials)** - -```bash -uv run pytest tests/test_databases/test_spiraldb_connector_integration.py -v --tb=short -``` - -Expected: all skipped (`Set SPIRAL_INTEGRATION_TESTS=1 to run SpiralDB integration tests`). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_databases/test_spiraldb_connector_integration.py -git commit -m "test(spiraldb): add TestArrowMetadataRoundTrip integration tests (ITL-471)" -``` - ---- - -## Task 8: Final check and push - -- [ ] **Step 1: Run the complete unit test suite** - -```bash -uv run pytest tests/ -v --tb=short -q --ignore=tests/test_databases/test_spiraldb_connector_integration.py -``` - -Expected: all pass. - -- [ ] **Step 2: Verify branch and push** - -```bash -git log --oneline -8 -git push -u origin eywalker/itl-471-spiraldbconnector-preserve-arrow-table-and-column-level -``` - ---- - -## Self-Review - -**Spec coverage check:** - -| Spec requirement | Task that covers it | -|---|---| -| `_serialize_arrow_metadata` helper | Task 2 | -| `_serialize_field_meta_tree` recursive helper | Task 2 | -| `_load_arrow_metadata` helper | Task 3 | -| `_restore_field` + `_restore_arrow_type` recursive helpers | Task 3 | -| `upsert_records` writes metadata after data write | Task 5 | -| `upsert_records` skips `set_metadata` when no metadata | Task 5 | -| `iter_batches` restores metadata per scan (once) | Task 6 | -| `iter_batches` backward compat (no stored metadata) | Task 6 | -| string/binary normalization preserved with metadata | Task 6 | -| `validate_records` on `DBConnectorProtocol` | Task 4 | -| `SQLiteConnector.validate_records` rejecting | Task 4 | -| `PostgreSQLConnector.validate_records` rejecting | Task 4 | -| `SpiralDBConnector.validate_records` no-op | Task 4 | -| `ConnectorArrowDatabase` guard replaced | Task 4 | -| Schema metadata integration test | Task 7 | -| Field metadata integration test | Task 7 | -| Extension type field metadata integration test | Task 7 | -| Nested struct field metadata integration test | Task 7 | -| Backward compatibility (no metadata) integration test | Task 7 | -| Full `ConnectorArrowDatabase` extension type round-trip | Task 7 | - -All spec requirements covered. ✅ - -**Placeholder scan:** No TBDs, no "similar to Task N" shortcuts, all code blocks complete. ✅ - -**Type consistency:** -- `_serialize_field_meta_tree` → `dict | None` (Tasks 2, 3) ✅ -- `_serialize_arrow_metadata` → `dict[str, bytes] | None` (Tasks 2, 5) ✅ -- `_load_arrow_metadata` → `tuple[dict[bytes, bytes] | None, dict[str, dict]]` (Tasks 3, 6) ✅ -- `_restore_field` → `tuple[pa.Field, bool]` (Tasks 3, 6) ✅ -- `_ARROW_METADATA_KEY` used consistently as `"__arrow_metadata__"` (Tasks 2, 3, 6, tests) ✅ diff --git a/superpowers/plans/2026-07-08-itl-508-indexed-entry-id-versioning.md b/superpowers/plans/2026-07-08-itl-508-indexed-entry-id-versioning.md deleted file mode 100644 index f6c141a38..000000000 --- a/superpowers/plans/2026-07-08-itl-508-indexed-entry-id-versioning.md +++ /dev/null @@ -1,1301 +0,0 @@ -# ITL-508: Indexed Entry ID Versioning Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the single-`entry_id` pipeline DB key with a `(base_entry_id, recomputation_index)` versioning scheme so that miss-triggered Phase 2 recomputation always writes a fresh pipeline record rather than being silently blocked by a stale duplicate key. - -**Architecture:** `compute_base_entry_id` (stable identity, used as in-memory cache key) + `compute_pipeline_entry_id(…, recomputation_index)` (versioned DB primary key). `add_pipeline_record` reads the max existing index for a `base_entry_id` and writes at `max+1`, so both single-threaded reruns and concurrent asyncio tasks eventually resolve to a valid pipeline record. Phase 1 filters the pipeline DB by `_PIPELINE_BASE_ENTRY_ID_COL` rather than the versioned key, so all recomputation-index versions are loaded and the inner-join naturally drops stale rows. - -**Tech Stack:** Python 3.12, PyArrow, Polars (for joins), asyncio TaskGroup, pytest-asyncio. - ---- - -## File Map - -| Role | Path | -|---|---| -| **Primary implementation** | `src/orcapod/core/nodes/function_node.py` | -| **Protocol update** | `src/orcapod/protocols/pipeline_protocols.py` | -| **New entry-id tests** | `tests/test_core/function_pod/test_function_node_caching.py` | -| **Ephemeral / recomputation tests** | `tests/test_core/function_pod/test_ephemeral_result.py` | -| **Fetch-joined tests** | `tests/test_core/nodes/test_function_node_fetch_joined.py` | -| **Get-cached tests** | `tests/test_core/nodes/test_function_node_get_cached.py` | - ---- - -## Task 1 — Add constants, `compute_base_entry_id`, and extend `compute_pipeline_entry_id` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (near line 73) -- Modify: `tests/test_core/function_pod/test_function_node_caching.py` - -- [ ] **Step 1: Write failing tests for the new functions** - -Add a `TestComputeBaseEntryId` class and a `TestVersionedEntryIdDiffersByIndex` class to `tests/test_core/function_pod/test_function_node_caching.py` immediately after the existing imports: - -```python -# --------------------------------------------------------------------------- -# compute_base_entry_id -# --------------------------------------------------------------------------- - - -class TestComputeBaseEntryId: - def test_returns_non_empty_bytes(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - data = Data({"x": 10}) - base_entry_id = node.compute_base_entry_id(tag, data) - assert isinstance(base_entry_id, bytes) - assert len(base_entry_id) > 0 - assert b":" in base_entry_id - - def test_same_inputs_produce_same_id(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - data = Data({"x": 10}) - assert node.compute_base_entry_id(tag, data) == node.compute_base_entry_id(tag, data) - - def test_different_tags_produce_different_ids(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - data = Data({"x": 10}) - assert node.compute_base_entry_id(Tag({"id": 0}), data) != node.compute_base_entry_id(Tag({"id": 1}), data) - - def test_different_data_produce_different_ids(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - assert node.compute_base_entry_id(tag, Data({"x": 10})) != node.compute_base_entry_id(tag, Data({"x": 99})) - - def test_differs_from_versioned_entry_id_at_index_zero(self): - """After ITL-508, compute_pipeline_entry_id includes the index in the preimage, - so compute_pipeline_entry_id(tag, data, 0) != compute_base_entry_id(tag, data).""" - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - data = Data({"x": 10}) - base_id = node.compute_base_entry_id(tag, data) - versioned_id_0 = node.compute_pipeline_entry_id(tag, data, 0) - assert base_id != versioned_id_0 - - -# --------------------------------------------------------------------------- -# compute_pipeline_entry_id with recomputation_index -# --------------------------------------------------------------------------- - - -class TestVersionedEntryIdDiffersByIndex: - def test_different_indices_produce_different_ids(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - data = Data({"x": 10}) - assert node.compute_pipeline_entry_id(tag, data, 0) != node.compute_pipeline_entry_id(tag, data, 1) - - def test_same_index_produces_same_id(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - data = Data({"x": 10}) - assert node.compute_pipeline_entry_id(tag, data, 1) == node.compute_pipeline_entry_id(tag, data, 1) - - def test_default_index_zero_is_consistent(self): - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag = Tag({"id": 0}) - data = Data({"x": 10}) - assert node.compute_pipeline_entry_id(tag, data) == node.compute_pipeline_entry_id(tag, data, 0) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -``` -uv run pytest tests/test_core/function_pod/test_function_node_caching.py::TestComputeBaseEntryId tests/test_core/function_pod/test_function_node_caching.py::TestVersionedEntryIdDiffersByIndex -v -``` - -Expected: `AttributeError: 'FunctionJobNode' object has no attribute 'compute_base_entry_id'` - -- [ ] **Step 3: Add the two new constants and `compute_base_entry_id` to `function_node.py`** - -Directly after line 73 (`_PIPELINE_ENTRY_ID_COL = "__pipeline_entry_id"`), add: - -```python -# Column storing the stable base entry ID (hash without recomputation index). -# Used for filtering in _fetch_joined_records and as the in-memory cache key. -_PIPELINE_BASE_ENTRY_ID_COL = "__pipeline_base_entry_id" - -# Column storing the recomputation chain index (pa.int32). -# 0 for the first computation, N+1 for each miss-triggered recompute. -_PIPELINE_RECOMPUTATION_INDEX_COL = "__pipeline_recomputation_index" -``` - -- [ ] **Step 4: Add `compute_base_entry_id` method to `FunctionJobNode`** - -Add immediately **before** the existing `compute_pipeline_entry_id` method (around line 1373): - -```python -def compute_base_entry_id( - self, tag: TagProtocol, input_data: DataProtocol -) -> bytes: - """Compute the stable base entry ID for a (tag, input_data, node) combination. - - Identical to the pre-ITL-508 ``compute_pipeline_entry_id`` implementation. - Stable across all recomputation attempts for the same logical input. - Used as the in-memory cache key (``_cached_output_datas``) and stored in - ``_PIPELINE_BASE_ENTRY_ID_COL``. - - Args: - tag: The tag (including system tags). - input_data: The input data. - - Returns: - Method-prefixed raw bytes (``b"{method}:{digest}"``) uniquely - identifying this (tag, input_data, node) combination. Suitable - for storage in a ``pa.large_binary()`` column. - """ - tag_with_hash = ( - tag.as_table(columns={"system_tags": True}) - .append_column( - constants.INPUT_DATA_HASH_COL, - pa.array([input_data.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - constants.NODE_CONTENT_HASH_COL, - pa.array([self.content_hash().to_string()], type=pa.large_string()), - ) - ) - return self.data_context.arrow_hasher.hash_table(tag_with_hash).to_prefixed_digest() -``` - -- [ ] **Step 5: Extend `compute_pipeline_entry_id` signature with `recomputation_index`** - -Replace the existing `compute_pipeline_entry_id` body (starting around line 1373) with: - -```python -def compute_pipeline_entry_id( - self, - tag: TagProtocol, - input_data: DataProtocol, - recomputation_index: int = 0, -) -> bytes: - """Compute a versioned pipeline entry ID from tag + system tags + input data hash + index. - - Appends a ``_PIPELINE_RECOMPUTATION_INDEX_COL`` column (value: - ``recomputation_index``, type ``pa.int32()``) to the hash preimage so - that each recomputation attempt receives a distinct DB primary key. - - At ``recomputation_index=0`` this produces a hash that differs from the - pre-ITL-508 implementation (the index column is now part of the preimage). - Existing pipeline DB records are implicitly invalidated — acceptable - because the project is pre-v0.1.0. - - ``NODE_CONTENT_HASH_COL`` is always included so that two runs processing - identical inputs each get a distinct entry ID, regardless of table scope. - - Args: - tag: The tag (including system tags). - input_data: The input data. - recomputation_index: Position in the recomputation chain. ``0`` for - the first computation, ``N+1`` for each miss-triggered recompute. - - Returns: - Method-prefixed raw bytes (``b"{method}:{digest}"``) uniquely - identifying this (tag, input_data, node run, recomputation attempt). - Suitable for storage in a ``pa.large_binary()`` column. - """ - tag_with_hash = ( - tag.as_table(columns={"system_tags": True}) - .append_column( - constants.INPUT_DATA_HASH_COL, - pa.array([input_data.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - constants.NODE_CONTENT_HASH_COL, - pa.array([self.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - _PIPELINE_RECOMPUTATION_INDEX_COL, - pa.array([recomputation_index], type=pa.int32()), - ) - ) - return self.data_context.arrow_hasher.hash_table(tag_with_hash).to_prefixed_digest() -``` - -- [ ] **Step 6: Run tests to confirm they pass** - -``` -uv run pytest tests/test_core/function_pod/test_function_node_caching.py::TestComputeBaseEntryId tests/test_core/function_pod/test_function_node_caching.py::TestVersionedEntryIdDiffersByIndex -v -``` - -Expected: all PASS - -- [ ] **Step 7: Confirm existing `TestComputePipelineEntryId` tests still pass** - -``` -uv run pytest tests/test_core/function_pod/test_function_node_caching.py::TestComputePipelineEntryId -v -``` - -Expected: all PASS (same-inputs-same-id, different-inputs-different-ids still hold at default index 0) - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/function_pod/test_function_node_caching.py -git commit -m "feat(ITL-508): add compute_base_entry_id and versioned compute_pipeline_entry_id" -``` - ---- - -## Task 2 — Redesign `add_pipeline_record` + remove `skip_cache_lookup` from protocol - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` -- Modify: `src/orcapod/protocols/pipeline_protocols.py` -- Modify: `tests/test_core/function_pod/test_ephemeral_result.py` - -- [ ] **Step 1: Write failing tests for the redesigned `add_pipeline_record`** - -Replace the entire `TestAddPipelineRecordDeduplication` class in `tests/test_core/function_pod/test_ephemeral_result.py` (lines 779–809) with: - -```python -# --------------------------------------------------------------------------- -# Task 2 tests: redesigned add_pipeline_record (indexed, no skip_cache_lookup) -# --------------------------------------------------------------------------- - - -class TestAddPipelineRecordIndexed: - def test_first_call_writes_at_index_zero(self): - """add_pipeline_record writes recomputation_index=0 on the first call.""" - from orcapod.core.nodes.function_node import _PIPELINE_RECOMPUTATION_INDEX_COL - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - node, _ = _make_node(stream, pipeline_db=pipeline_db) - tag, data = next(iter(stream.iter_data())) - - node.add_pipeline_record(tag, data, data_record_id=uuid.uuid4(), computed=True) - pipeline_db.flush() - - all_records = pipeline_db.get_all_records(node.node_identity_path) - assert all_records is not None - assert all_records.num_rows == 1 - assert all_records.column(_PIPELINE_RECOMPUTATION_INDEX_COL)[0].as_py() == 0 - - def test_second_call_writes_at_index_one(self): - """Second add_pipeline_record call for the same base_entry_id writes at index 1.""" - from orcapod.core.nodes.function_node import _PIPELINE_RECOMPUTATION_INDEX_COL - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - node, _ = _make_node(stream, pipeline_db=pipeline_db) - tag, data = next(iter(stream.iter_data())) - - node.add_pipeline_record(tag, data, data_record_id=uuid.uuid4(), computed=True) - pipeline_db.flush() - node.add_pipeline_record(tag, data, data_record_id=uuid.uuid4(), computed=True) - pipeline_db.flush() - - all_records = pipeline_db.get_all_records(node.node_identity_path) - assert all_records is not None - assert all_records.num_rows == 2 - indices = all_records.column(_PIPELINE_RECOMPUTATION_INDEX_COL).to_pylist() - assert sorted(indices) == [0, 1] - - def test_base_entry_id_column_written(self): - """add_pipeline_record writes _PIPELINE_BASE_ENTRY_ID_COL to the pipeline DB row.""" - from orcapod.core.nodes.function_node import _PIPELINE_BASE_ENTRY_ID_COL - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - node, _ = _make_node(stream, pipeline_db=pipeline_db) - tag, data = next(iter(stream.iter_data())) - - node.add_pipeline_record(tag, data, data_record_id=uuid.uuid4(), computed=True) - pipeline_db.flush() - - all_records = pipeline_db.get_all_records(node.node_identity_path) - assert all_records is not None - assert _PIPELINE_BASE_ENTRY_ID_COL in all_records.column_names - expected_base_id = node.compute_base_entry_id(tag, data) - assert all_records.column(_PIPELINE_BASE_ENTRY_ID_COL)[0].as_py() == expected_base_id - - def test_skip_cache_lookup_parameter_removed(self): - """add_pipeline_record no longer accepts skip_cache_lookup — raises TypeError.""" - stream = _make_stream([{"id": 0, "x": 10}]) - node, _ = _make_node(stream) - tag, data = next(iter(stream.iter_data())) - with pytest.raises(TypeError): - node.add_pipeline_record( - tag, data, data_record_id=uuid.uuid4(), computed=True, - skip_cache_lookup=False, # removed parameter - ) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestAddPipelineRecordIndexed -v -``` - -Expected: `FAILED` — `_PIPELINE_RECOMPUTATION_INDEX_COL` doesn't exist yet in pipeline DB rows; `skip_cache_lookup` still accepted. - -- [ ] **Step 3: Redesign `add_pipeline_record` in `function_node.py`** - -Replace the entire `add_pipeline_record` method (lines 1405–1483): - -```python -def add_pipeline_record( - self, - tag: TagProtocol, - input_data: DataProtocol, - data_record_id: uuid.UUID, - computed: bool, - is_ephemeral: bool = False, -) -> None: - """Add a pipeline record to the database for a processed data. - - Computes the next recomputation index by querying existing rows in the - pipeline DB that share the same ``base_entry_id``, then writes at - ``max_index + 1``. The write uses ``skip_duplicates=True`` so that - concurrent asyncio coroutines competing for the same versioned entry ID - are safely serialised: the first writer lands, subsequent writers whose - compute happened to produce the same hash are silently no-oped. - - The pipeline record stores: - - - Tag columns (including system tags) - - All source columns of the input data (provenance, not data values) - - Output data record ID (for joining with result records) - - Base entry ID (``_PIPELINE_BASE_ENTRY_ID_COL``) - - Recomputation index (``_PIPELINE_RECOMPUTATION_INDEX_COL``) - - Whether the result is stored in the ephemeral store - - Input data context key - - Whether the result was freshly computed or cached - - Args: - tag: The tag associated with the input data. - input_data: The input data that was processed. - data_record_id: UUID of the result record in the result database. - computed: Whether the result was freshly computed (``True``) or - served from a cache (``False``). - is_ephemeral: Whether the result is stored in the ephemeral store. - """ - self._require_pipeline_database() - base_entry_id = self.compute_base_entry_id(tag, input_data) - - # Determine the next recomputation index by reading all existing rows - # that share this base_entry_id. Steps 1–5 below are fully synchronous - # (no await), so concurrent asyncio coroutines serialise naturally. - existing = self._pipeline_database.get_records_with_column_value( - self.node_identity_path, - {_PIPELINE_BASE_ENTRY_ID_COL: base_entry_id}, - ) - if existing is None or existing.num_rows == 0: - new_index = 0 - else: - indices = existing.column(_PIPELINE_RECOMPUTATION_INDEX_COL).to_pylist() - new_index = max(indices) + 1 - - versioned_entry_id = self.compute_pipeline_entry_id(tag, input_data, new_index) - - # Extract source columns only (no data columns) from the input data - input_table_with_source = input_data.as_table(columns={"source": True}) - source_col_names = [ - c - for c in input_table_with_source.column_names - if c.startswith(constants.SOURCE_PREFIX) - ] - input_source_table = input_table_with_source.select(source_col_names) - - # Build the meta columns table - meta_table = pa.table( - { - constants.DATA_RECORD_ID: pa.array( - [data_record_id.bytes], type=pa.large_binary() - ), - constants.NODE_CONTENT_HASH_COL: pa.array( - [self.content_hash().to_string()], type=pa.large_string() - ), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array( - [input_data.data_context_key], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array( - [computed], type=pa.bool_() - ), - constants.IS_EPHEMERAL_COL: pa.array( - [is_ephemeral], type=pa.bool_() - ), - _PIPELINE_BASE_ENTRY_ID_COL: pa.array( - [base_entry_id], type=pa.large_binary() - ), - _PIPELINE_RECOMPUTATION_INDEX_COL: pa.array( - [new_index], type=pa.int32() - ), - } - ) - - # Combine: tag (with system tags) + input source columns + meta columns - combined_record = arrow_utils.hstack_tables( - tag.as_table(columns={"system_tags": True}), - input_source_table, - meta_table, - ) - - self._pipeline_database.add_record( - self.node_identity_path, - versioned_entry_id, - combined_record, - skip_duplicates=True, - ) -``` - -- [ ] **Step 4: Remove `skip_cache_lookup` from `pipeline_protocols.py`** - -In `src/orcapod/protocols/pipeline_protocols.py`, replace: - -```python - def add_pipeline_record( - self, - tag: cp.TagProtocol, - input_data: cp.DataProtocol, - data_record_id: str, - retrieved: bool | None = None, - skip_cache_lookup: bool = False, - ) -> None: ... -``` - -with: - -```python - def add_pipeline_record( - self, - tag: cp.TagProtocol, - input_data: cp.DataProtocol, - data_record_id: str, - retrieved: bool | None = None, - ) -> None: ... -``` - -- [ ] **Step 5: Run tests to confirm they pass** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestAddPipelineRecordIndexed -v -``` - -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py src/orcapod/protocols/pipeline_protocols.py tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "feat(ITL-508): redesign add_pipeline_record with recomputation index; remove skip_cache_lookup" -``` - ---- - -## Task 3 — Update Phase 2 callers: `_process_data_internal` and `_async_process_data_internal` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` - -These two methods need two changes each: -1. Remove `skip_cache_lookup=True` from `add_pipeline_record` calls (parameter removed). -2. Switch `_cached_output_datas` key from `compute_pipeline_entry_id` to `compute_base_entry_id`. - -- [ ] **Step 1: Verify that the pre-written TDD test currently FAILS** - -The test at line 631 of `test_ephemeral_result.py` describes session 3 NOT recomputing after a persistent miss + recompute. With the old code it currently fails because the pipeline record at index 1 was never written. - -``` -uv run pytest "tests/test_core/function_pod/test_ephemeral_result.py::TestPersistentMissWarning::test_recompute_after_persistent_miss_appends_new_pipeline_record" -v -``` - -Expected: `FAILED` (session 3 recomputes; call_count reaches 3 instead of staying at 2) - -Also verify the ephemeral-miss cycle test currently fails: - -``` -uv run pytest "tests/test_core/function_pod/test_ephemeral_result.py::TestEphemeralWritePath::test_recompute_after_ephemeral_miss_no_infinite_cycle" -v -``` - -Expected: `FAILED` - -- [ ] **Step 2: Update `_process_data_internal`** - -In `_process_data_internal`, make two sets of changes: - -**a) Ephemeral branch** — remove `skip_cache_lookup=True`: - -Replace: -```python - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, - is_ephemeral=True, - ) -``` - -With: -```python - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - is_ephemeral=True, - ) -``` - -**b) Persistent branch** — remove `skip_cache_lookup=True`: - -Replace: -```python - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, - ) -``` - -With: -```python - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - ) -``` - -**c) Cache write** — switch key from versioned to base: - -Replace: -```python - # Store by entry_id and invalidate derived caches - entry_id = self.compute_pipeline_entry_id(tag, data) - self._cached_output_datas[entry_id] = (tag_out, output_data) -``` - -With: -```python - # Store by base_entry_id (stable across recomputation cycles) and invalidate caches - base_entry_id = self.compute_base_entry_id(tag, data) - self._cached_output_datas[base_entry_id] = (tag_out, output_data) -``` - -- [ ] **Step 3: Update `_async_process_data_internal`** (identical changes) - -**a) Ephemeral branch** — remove `skip_cache_lookup=True`: - -Replace: -```python - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, - is_ephemeral=True, - ) -``` - -With: -```python - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - is_ephemeral=True, - ) -``` - -**b) Persistent branch** — remove `skip_cache_lookup=True`: - -Replace: -```python - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - skip_cache_lookup=True, - ) -``` - -With: -```python - self.add_pipeline_record( - tag, - data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - ) -``` - -**c) Cache write** — switch key from versioned to base: - -Replace: -```python - # Store by entry_id and invalidate derived caches - entry_id = self.compute_pipeline_entry_id(tag, data) - self._cached_output_datas[entry_id] = (tag_out, output_data) -``` - -With: -```python - # Store by base_entry_id (stable across recomputation cycles) and invalidate caches - base_entry_id = self.compute_base_entry_id(tag, data) - self._cached_output_datas[base_entry_id] = (tag_out, output_data) -``` - -- [ ] **Step 4: Run the pre-written TDD tests to confirm they now pass** - -``` -uv run pytest "tests/test_core/function_pod/test_ephemeral_result.py::TestPersistentMissWarning::test_recompute_after_persistent_miss_appends_new_pipeline_record" "tests/test_core/function_pod/test_ephemeral_result.py::TestEphemeralWritePath::test_recompute_after_ephemeral_miss_no_infinite_cycle" -v -``` - -Expected: both PASS - -- [ ] **Step 5: Run the full ephemeral test suite to verify no regressions** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py -v -``` - -Expected: all PASS (some tests that previously called `add_pipeline_record` with `skip_cache_lookup` will now fail — those are addressed in Task 5's regression fixes) - -Note: `TestAddPipelineRecord` tests use `node.execute(stream)` not direct `add_pipeline_record` calls — they will still pass. The only broken test is the now-deleted `TestAddPipelineRecordDeduplication.test_duplicate_not_added_when_skip_cache_lookup_false` (replaced in Task 2). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "feat(ITL-508): switch Phase 2 cache key to base_entry_id; remove skip_cache_lookup from callers" -``` - ---- - -## Task 4 — Update Phase 1 internals: `get_cached_results`, `_fetch_joined_records`, `_load_cached_entries`, `get_all_records` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` -- Modify: `tests/test_core/nodes/test_function_node_fetch_joined.py` -- Modify: `tests/test_core/nodes/test_function_node_get_cached.py` - -- [ ] **Step 1: Write failing tests for the renamed parameter in `_fetch_joined_records`** - -In `tests/test_core/nodes/test_function_node_fetch_joined.py`, update the import at line 11: - -Replace: -```python -from orcapod.core.nodes.function_node import FunctionJobNode, _PIPELINE_ENTRY_ID_COL -``` - -With: -```python -from orcapod.core.nodes.function_node import ( - FunctionJobNode, - _PIPELINE_ENTRY_ID_COL, - _PIPELINE_BASE_ENTRY_ID_COL, -) -``` - -Then update `test_entry_ids_filter_narrows_rows` (currently lines 97–105): - -Replace: -```python - def test_entry_ids_filter_narrows_rows(self, executed_node): - """Passing a single entry_id returns only that row.""" - node = executed_node - input_pairs = list(node._input_stream.iter_data()) - entry_id_0 = node.compute_pipeline_entry_id(input_pairs[0][0], input_pairs[0][1]) - - result = node._fetch_joined_records(entry_ids=[entry_id_0]) - assert result is not None - assert result.table.num_rows == 1 - assert result.table.column(_PIPELINE_ENTRY_ID_COL)[0].as_py() == entry_id_0 -``` - -With: -```python - def test_entry_ids_filter_narrows_rows(self, executed_node): - """Passing a single base_entry_id returns only that row.""" - node = executed_node - input_pairs = list(node._input_stream.iter_data()) - base_entry_id_0 = node.compute_base_entry_id(input_pairs[0][0], input_pairs[0][1]) - - result = node._fetch_joined_records(base_entry_ids=[base_entry_id_0]) - assert result is not None - assert result.table.num_rows == 1 - assert result.table.column(_PIPELINE_BASE_ENTRY_ID_COL)[0].as_py() == base_entry_id_0 -``` - -And update `test_empty_entry_ids_returns_zero_rows`: - -Replace: -```python - result = executed_node._fetch_joined_records(entry_ids=[]) -``` - -With: -```python - result = executed_node._fetch_joined_records(base_entry_ids=[]) -``` - -- [ ] **Step 2: Write failing tests for the renamed parameter in `get_cached_results`** - -In `tests/test_core/nodes/test_function_node_get_cached.py`, update all tests that call `compute_pipeline_entry_id` and `get_cached_results(entry_ids)`: - -Replace the three tests `test_returns_cached_results_for_matching_entry_ids`, `test_filters_to_requested_entry_ids_only`, and `test_get_cached_results_populates_internal_cache` entirely: - -```python - def test_returns_cached_results_for_matching_entry_ids(self, function_node_with_db): - node = function_node_with_db - all_pairs = list(node._input_stream.iter_data()) - - base_entry_ids = [] - for tag, data in all_pairs: - node.execute_data(tag, data) - base_entry_ids.append(node.compute_base_entry_id(tag, data)) - - cached = node.get_cached_results(base_entry_ids) - assert len(cached) == 2 - assert all(eid in cached for eid in base_entry_ids) - - def test_filters_to_requested_entry_ids_only(self, function_node_with_db): - node = function_node_with_db - all_pairs = list(node._input_stream.iter_data()) - - base_entry_ids = [] - for tag, data in all_pairs: - node.execute_data(tag, data) - base_entry_ids.append(node.compute_base_entry_id(tag, data)) - - cached = node.get_cached_results([base_entry_ids[0]]) - assert len(cached) == 1 - assert base_entry_ids[0] in cached - assert base_entry_ids[1] not in cached - - def test_get_cached_results_populates_internal_cache(self, function_node_with_db): - """get_cached_results should populate _cached_output_datas keyed by base_entry_id.""" - node = function_node_with_db - all_pairs = list(node._input_stream.iter_data()) - - base_entry_ids = [] - for tag, data in all_pairs: - node.execute_data(tag, data) - base_entry_ids.append(node.compute_base_entry_id(tag, data)) - - # Clear internal cache - node._cached_output_datas.clear() - assert len(node._cached_output_datas) == 0 - - # get_cached_results should repopulate - node.get_cached_results(base_entry_ids) - assert len(node._cached_output_datas) == 2 -``` - -- [ ] **Step 3: Run tests to confirm they fail** - -``` -uv run pytest tests/test_core/nodes/test_function_node_fetch_joined.py tests/test_core/nodes/test_function_node_get_cached.py -v -``` - -Expected: several `FAILED` — parameter name mismatch or `_PIPELINE_BASE_ENTRY_ID_COL` not present in DB rows yet. - -- [ ] **Step 4: Update `get_cached_results` in `function_node.py`** - -Replace the method signature and body (lines ~1254–1293): - -```python -def get_cached_results( - self, base_entry_ids: list[bytes] -) -> dict[bytes, tuple[TagProtocol, DataProtocol]]: - """Public cache façade: return already-computed results for the given base entry IDs. - - Serves hits directly from the in-memory cache (``_cached_output_datas``). - For IDs not yet cached, delegates to ``_load_cached_entries`` which calls - ``_fetch_joined_records`` to load from the pipeline and result databases. - Add-only semantics: existing in-memory entries are never cleared or - overwritten. - - Args: - base_entry_ids: Stable base entry IDs (from ``compute_base_entry_id``) - to look up. - - Returns: - Mapping from base_entry_id to ``(tag, output_data)`` for found entries. - Empty dict if no DB is attached, ``base_entry_ids`` is empty, or no - matches are found. - """ - if self._cached_function_pod is None or not base_entry_ids: - return {} - - missing = [eid for eid in base_entry_ids if eid not in self._cached_output_datas] - if missing: - loaded = self._load_cached_entries(base_entry_ids=missing) - self._cached_output_datas.update(loaded) - if loaded: - self._cached_output_table = None - self._cached_content_hash_column = None - - return { - eid: self._cached_output_datas[eid] - for eid in base_entry_ids - if eid in self._cached_output_datas - and self._cached_output_datas[eid][1] is not None - } -``` - -- [ ] **Step 5: Update `_fetch_joined_records` in `function_node.py`** - -**a) Rename the parameter** (line ~1582): - -Replace: -```python - def _fetch_joined_records( - self, - entry_ids: list[bytes] | None = None, - ) -> _JoinedRecords | None: -``` - -With: -```python - def _fetch_joined_records( - self, - base_entry_ids: list[bytes] | None = None, - ) -> _JoinedRecords | None: -``` - -Also update the docstring parameter description: change `entry_ids` → `base_entry_ids` and `_PIPELINE_ENTRY_ID_COL` → `_PIPELINE_BASE_ENTRY_ID_COL` in the description. - -**b) Update the anti-join** to use `_PIPELINE_BASE_ENTRY_ID_COL` (lines ~1698–1704): - -Replace: -```python - if ephemeral_df.height > 0 and persistent_df.height > 0: - ephemeral_only_df = ephemeral_df.join( - persistent_df.select([_PIPELINE_ENTRY_ID_COL]), - on=_PIPELINE_ENTRY_ID_COL, - how="anti", - ) -``` - -With: -```python - if ephemeral_df.height > 0 and persistent_df.height > 0: - ephemeral_only_df = ephemeral_df.join( - persistent_df.select([_PIPELINE_BASE_ENTRY_ID_COL]), - on=_PIPELINE_BASE_ENTRY_ID_COL, - how="anti", - ) -``` - -**c) Update the filter** to use `_PIPELINE_BASE_ENTRY_ID_COL` (lines ~1714–1718): - -Replace: -```python - # Apply entry_id filter if requested - if entry_ids is not None: - merged_df = merged_df.filter( - pl.col(_PIPELINE_ENTRY_ID_COL).is_in(entry_ids) - ) -``` - -With: -```python - # Apply base_entry_id filter if requested - if base_entry_ids is not None: - merged_df = merged_df.filter( - pl.col(_PIPELINE_BASE_ENTRY_ID_COL).is_in(base_entry_ids) - ) -``` - -- [ ] **Step 6: Update `_load_cached_entries` in `function_node.py`** - -**a) Rename the parameter** (line ~1728): - -Replace: -```python - def _load_cached_entries( - self, - entry_ids: list[bytes] | None = None, - ) -> "dict[bytes, tuple[TagProtocol, DataProtocol]]": -``` - -With: -```python - def _load_cached_entries( - self, - base_entry_ids: list[bytes] | None = None, - ) -> "dict[bytes, tuple[TagProtocol, DataProtocol]]": -``` - -**b) Update the internal `_fetch_joined_records` call** (line ~1755): - -Replace: -```python - fetched = self._fetch_joined_records(entry_ids=entry_ids) -``` - -With: -```python - fetched = self._fetch_joined_records(base_entry_ids=base_entry_ids) -``` - -**c) Update the column extraction and dict key** (lines ~1781–1793): - -Replace: -```python - entry_ids_col = joined.column(_PIPELINE_ENTRY_ID_COL).to_pylist() - drop_cols = [ - c - for c in joined.column_names - if c.startswith(constants.META_PREFIX) - or c == constants.NODE_CONTENT_HASH_COL - ] - data_table = joined.drop([c for c in drop_cols if c in joined.column_names]) - stream = ArrowTableStream(data_table, tag_columns=tag_keys) - - loaded: dict[bytes, tuple[TagProtocol, DataProtocol]] = {} - for eid, (tag, data) in zip(entry_ids_col, stream.iter_data()): - loaded[eid] = (tag, data) - return loaded -``` - -With: -```python - base_entry_ids_col = joined.column(_PIPELINE_BASE_ENTRY_ID_COL).to_pylist() - drop_cols = [ - c - for c in joined.column_names - if c.startswith(constants.META_PREFIX) - or c == constants.NODE_CONTENT_HASH_COL - ] - data_table = joined.drop([c for c in drop_cols if c in joined.column_names]) - stream = ArrowTableStream(data_table, tag_columns=tag_keys) - - loaded: dict[bytes, tuple[TagProtocol, DataProtocol]] = {} - for base_eid, (tag, data) in zip(base_entry_ids_col, stream.iter_data()): - loaded[base_eid] = (tag, data) - return loaded -``` - -- [ ] **Step 7: Update `get_all_records` to always-drop the new internal columns** - -In `get_all_records`, replace (lines ~1527–1528): - -```python - drop_columns = [constants.NODE_CONTENT_HASH_COL, _PIPELINE_ENTRY_ID_COL] -``` - -With: - -```python - drop_columns = [ - constants.NODE_CONTENT_HASH_COL, - _PIPELINE_ENTRY_ID_COL, - _PIPELINE_BASE_ENTRY_ID_COL, - _PIPELINE_RECOMPUTATION_INDEX_COL, - ] -``` - -- [ ] **Step 8: Run tests to confirm they pass** - -``` -uv run pytest tests/test_core/nodes/test_function_node_fetch_joined.py tests/test_core/nodes/test_function_node_get_cached.py -v -``` - -Expected: all PASS - -- [ ] **Step 9: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/nodes/test_function_node_fetch_joined.py tests/test_core/nodes/test_function_node_get_cached.py -git commit -m "feat(ITL-508): re-key Phase 1 internals to base_entry_id; update fetch/load/get_cached" -``` - ---- - -## Task 5 — Update `execute()` and `async_execute()` Phase 1 callers - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` - -- [ ] **Step 1: Update `execute()` to use `compute_base_entry_id`** - -In `execute()` (around lines 1112–1131), replace: - -```python - # Collect upstream entries and resolve entry_ids - upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ - (tag, data, self.compute_pipeline_entry_id(tag, data)) - for tag, data in input_stream.iter_data() - ] - entry_ids = [eid for _, _, eid in upstream_entries] - - # Hot-load any already-computed results from DB into _cached_output_datas. - # get_cached_results() is called for its side effect (populating the - # in-memory cache); the returned dict is intentionally discarded here so - # that the per-data cache-hit check below uses _cached_output_datas - # directly — which includes None-output entries (function returned None) - # and prevents spurious recomputation of already-processed data. - self.get_cached_results(entry_ids=entry_ids) - - output: list[tuple[TagProtocol, DataProtocol]] = [] - for tag, data, entry_id in upstream_entries: - ctx_obs.on_data_start(node_label, tag, data) - - if entry_id in self._cached_output_datas: - tag_out, result = self._cached_output_datas[entry_id] -``` - -With: - -```python - # Collect upstream entries and resolve base_entry_ids (stable across recomputation) - upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ - (tag, data, self.compute_base_entry_id(tag, data)) - for tag, data in input_stream.iter_data() - ] - base_entry_ids = [eid for _, _, eid in upstream_entries] - - # Hot-load any already-computed results from DB into _cached_output_datas. - # get_cached_results() is called for its side effect (populating the - # in-memory cache); the returned dict is intentionally discarded here so - # that the per-data cache-hit check below uses _cached_output_datas - # directly — which includes None-output entries (function returned None) - # and prevents spurious recomputation of already-processed data. - self.get_cached_results(base_entry_ids=base_entry_ids) - - output: list[tuple[TagProtocol, DataProtocol]] = [] - for tag, data, base_entry_id in upstream_entries: - ctx_obs.on_data_start(node_label, tag, data) - - if base_entry_id in self._cached_output_datas: - tag_out, result = self._cached_output_datas[base_entry_id] -``` - -- [ ] **Step 2: Update `async_execute()` DB path to use `compute_base_entry_id`** - -In `async_execute()` (around lines 1984–2004), replace: - -```python - cached_by_entry_id: dict[bytes, tuple[TagProtocol, DataProtocol]] = dict(loaded) - - # Phase 2: drive output from input channel — cached or compute - async def _process_one_db( - tag: TagProtocol, data: DataProtocol - ) -> None: - entry_id = self.compute_pipeline_entry_id(tag, data) - if entry_id in cached_by_entry_id: - tag_out, result_data = cached_by_entry_id[entry_id] -``` - -With: - -```python - cached_by_base_entry_id: dict[bytes, tuple[TagProtocol, DataProtocol]] = dict(loaded) - - # Phase 2: drive output from input channel — cached or compute - async def _process_one_db( - tag: TagProtocol, data: DataProtocol - ) -> None: - base_entry_id = self.compute_base_entry_id(tag, data) - if base_entry_id in cached_by_base_entry_id: - tag_out, result_data = cached_by_base_entry_id[base_entry_id] -``` - -- [ ] **Step 3: Run the full caching and ephemeral test suites** - -``` -uv run pytest tests/test_core/function_pod/test_function_node_caching.py tests/test_core/function_pod/test_ephemeral_result.py -v -``` - -Expected: all PASS - -- [ ] **Step 4: Run the node protocol tests** - -``` -uv run pytest tests/test_pipeline/test_node_protocols.py -v -``` - -Expected: all PASS (no changes needed — mock classes don't test `add_pipeline_record` signature) - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "feat(ITL-508): update execute() and async_execute() Phase 1 to use base_entry_id" -``` - ---- - -## Task 6 — Add concurrent-miss asyncio test + run full suite - -**Files:** -- Modify: `tests/test_core/function_pod/test_ephemeral_result.py` - -- [ ] **Step 1: Add `TestConcurrentMissSerialization` class** - -Add at the end of `tests/test_core/function_pod/test_ephemeral_result.py`: - -```python -# --------------------------------------------------------------------------- -# Task 6 tests: concurrent asyncio Phase 2 serialisation -# --------------------------------------------------------------------------- - - -class TestConcurrentMissSerialization: - @pytest.mark.asyncio - async def test_two_concurrent_phase2_misses_produce_valid_pipeline_records(self): - """Two asyncio coroutines that simultaneously execute Phase 2 for the same input - each produce a valid pipeline record. A subsequent Phase 1 lookup finds a result - and does NOT recompute.""" - call_count = {"n": 0} - - def counting_double(x: int) -> int: - call_count["n"] += 1 - return x * 2 - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - pf = PythonDataFunction(counting_double, output_keys="result") - pod = FunctionPod(pf) - node = FunctionJobNode( - function_pod=pod, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db, - ) - - tag, data = next(iter(stream.iter_data())) - - # Two concurrent _async_process_data_internal calls on the same (tag, data). - # In asyncio cooperative multitasking, these serialise at add_pipeline_record - # (synchronous), so each gets a distinct recomputation_index. - async with asyncio.TaskGroup() as tg: - tg.create_task(node._async_process_data_internal(tag, data)) - tg.create_task(node._async_process_data_internal(tag, data)) - - # At least one pipeline record must exist for this base_entry_id - pipeline_db.flush() - all_records = pipeline_db.get_all_records(node.node_identity_path) - assert all_records is not None - assert all_records.num_rows >= 1 - - # Session 2: new node with the same DBs — Phase 1 must find a valid result - pf2 = PythonDataFunction(counting_double, output_keys="result") - pod2 = FunctionPod(pf2) - node2 = FunctionJobNode( - function_pod=pod2, - input_stream=stream, - pipeline_database=pipeline_db, - result_database=result_db, - ) - count_before_session2 = call_count["n"] - results = node2.execute(stream) - assert len(results) == 1 - assert results[0][1].as_dict()["result"] == 20 - assert call_count["n"] == count_before_session2 # NOT recomputed - - @pytest.mark.asyncio - async def test_sequential_add_pipeline_record_increments_index_each_time(self): - """Two sequential add_pipeline_record calls for the same base_entry_id - write at indices 0 and 1 respectively (not blocked by the existing row).""" - from orcapod.core.nodes.function_node import ( - _PIPELINE_BASE_ENTRY_ID_COL, - _PIPELINE_RECOMPUTATION_INDEX_COL, - ) - - stream = _make_stream([{"id": 0, "x": 10}]) - pipeline_db = InMemoryArrowDatabase() - node, _ = _make_node(stream, pipeline_db=pipeline_db) - tag, data = next(iter(stream.iter_data())) - - node.add_pipeline_record(tag, data, data_record_id=uuid.uuid4(), computed=True) - node.add_pipeline_record(tag, data, data_record_id=uuid.uuid4(), computed=True) - pipeline_db.flush() - - all_records = pipeline_db.get_all_records(node.node_identity_path) - assert all_records is not None - assert all_records.num_rows == 2 - - from orcapod.core.nodes.function_node import _PIPELINE_BASE_ENTRY_ID_COL - base_ids = all_records.column(_PIPELINE_BASE_ENTRY_ID_COL).to_pylist() - assert base_ids[0] == base_ids[1] # same base_entry_id - - indices = all_records.column(_PIPELINE_RECOMPUTATION_INDEX_COL).to_pylist() - assert sorted(indices) == [0, 1] # distinct indices -``` - -- [ ] **Step 2: Run the new concurrent tests** - -``` -uv run pytest tests/test_core/function_pod/test_ephemeral_result.py::TestConcurrentMissSerialization -v -``` - -Expected: both PASS - -- [ ] **Step 3: Run the full test suite** - -``` -uv run pytest tests/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: all tests pass, no regressions. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_core/function_pod/test_ephemeral_result.py -git commit -m "test(ITL-508): add concurrent-miss asyncio serialisation tests" -``` - ---- - -## Self-Review - -### Spec coverage check - -| Spec section | Task that covers it | -|---|---| -| §1 New pipeline DB columns (`_PIPELINE_BASE_ENTRY_ID_COL`, `_PIPELINE_RECOMPUTATION_INDEX_COL`) | Task 2 Step 3 | -| §2 `compute_base_entry_id` | Task 1 Steps 3–4 | -| §2 Extended `compute_pipeline_entry_id` | Task 1 Step 5 | -| §3 In-memory cache re-keying (`_cached_output_datas` → `base_entry_id`) | Task 3 Steps 2–3 | -| §4 Phase 1 `execute()` and `async_execute()` | Task 5 Steps 1–2 | -| §4 `get_cached_results(base_entry_ids)` | Task 4 Step 4 | -| §4 `_fetch_joined_records(base_entry_ids)` + filter on BASE col | Task 4 Step 5 | -| §4 `_load_cached_entries` keyed by BASE col | Task 4 Step 6 | -| §5 `add_pipeline_record` redesign (remove `skip_cache_lookup`, max+1 index) | Task 2 Step 3 | -| §5 Phase 2 callers updated | Task 3 Steps 2–3 | -| §6 Anti-join on `_PIPELINE_BASE_ENTRY_ID_COL` (persistent wins over ephemeral) | Task 4 Step 5b | -| §7 Protocol update (remove `skip_cache_lookup`) | Task 2 Step 4 | -| Testing: single-threaded correctness | Pre-written in `test_ephemeral_result.py` (Task 3 Step 4) | -| Testing: concurrent-miss serialisation | Task 6 Step 1 | -| Testing: ephemeral + persistent coexistence | Existing `test_persistent_result_outcompetes_ephemeral` | -| Testing: regression | Task 6 Step 3 | - -### Placeholder scan - -No TBD or TODO markers. All steps contain exact code. - -### Type consistency - -- `compute_base_entry_id` returns `bytes` — consistent with `compute_pipeline_entry_id` return type. -- `_PIPELINE_BASE_ENTRY_ID_COL` is `pa.large_binary()` in the pipeline record — consistent with `base_entry_id: bytes` type used in `get_records_with_column_value` filter and `is_in` filter. -- `_PIPELINE_RECOMPUTATION_INDEX_COL` is `pa.int32()` — `max(indices)` where `indices` is a Python `list[int]` from `.to_pylist()` — consistent. -- `get_cached_results(base_entry_ids: list[bytes])` — callers in `execute()` and `async_execute()` pass `base_entry_ids` built from `compute_base_entry_id` which returns `bytes` — consistent. -- `_fetch_joined_records(base_entry_ids: list[bytes] | None)` — callers pass either `None` (load all) or a `list[bytes]` from `compute_base_entry_id` — consistent. -- `_load_cached_entries(base_entry_ids: list[bytes] | None)` — same as above — consistent. diff --git a/superpowers/plans/2026-07-08-itl-511-directory-hash-caching.md b/superpowers/plans/2026-07-08-itl-511-directory-hash-caching.md deleted file mode 100644 index 12fa61f5e..000000000 --- a/superpowers/plans/2026-07-08-itl-511-directory-hash-caching.md +++ /dev/null @@ -1,836 +0,0 @@ -# Directory Hash Caching (ITL-511) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Wire `BasicDirectoryHasher` to use an injected `FileContentHasherProtocol` for file-leaf hashing, extend `enable_file_hash_caching()` to also patch `DirectoryHandler`, and add user-facing documentation. - -**Architecture:** `BasicDirectoryHasher` gains a required `file_hasher: FileContentHasherProtocol` constructor argument; `_hash_dir()` calls `file_hasher.hash_file()` instead of the bare `hash_utils.hash_file()`. `enable_file_hash_caching()` creates a shared `CachedFileHasher` and registers it with both `FileHandler` and a new `BasicDirectoryHasher` inside `DirectoryHandler`. `v0.1.json` wires the context-level `file_hasher` ref into `BasicDirectoryHasher`. - -**Tech Stack:** Python 3.12, `uv run pytest`, `sqlite3`, `upath` - -**Spec:** `superpowers/specs/2026-07-08-itl-511-directory-hash-caching-design.md` - -**Branch:** `eywalker/itl-511-verify-opdirectory-hashing-uses-filehasher-cache-document` - ---- - -## Task 0: Create and check out the feature branch - -**Files:** none - -- [ ] **Step 0.1: Create the branch** - -```bash -git checkout main -git pull -git checkout -b eywalker/itl-511-verify-opdirectory-hashing-uses-filehasher-cache-document -git branch --show-current -``` - -Expected: `eywalker/itl-511-verify-opdirectory-hashing-uses-filehasher-cache-document` - ---- - -## Task 1: Write failing caching integration tests - -These tests verify the fix before it exists. They will fail because `BasicDirectoryHasher` -does not yet accept a `file_hasher` argument. - -**Files:** -- Create: `tests/test_hashing/test_directory_hash_caching.py` - -- [ ] **Step 1.1: Create the test file** - -Create `tests/test_hashing/test_directory_hash_caching.py` with the following content: - -```python -"""Integration tests: op.Directory hashing with CachedFileHasher. - -Verifies that BasicDirectoryHasher consults the file-hash cache at the -per-file level during Merkle tree traversal, and that -enable_file_hash_caching() wires the same CachedFileHasher instance into -both FileHandler and DirectoryHandler. -""" - -from __future__ import annotations - -import pytest - -from orcapod.extension_types.directory_type import Directory -from orcapod.extension_types.file_type import File -from orcapod.hashing.directory_hashers import BasicDirectoryHasher -from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher -from orcapod.hashing.hash_cachers import InMemoryHashCacher -from orcapod.types import ContentHash - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -class CountingFileHasher: - """FileContentHasherProtocol wrapper that counts hash_file() calls.""" - - def __init__(self, inner: FileHasher) -> None: - self.inner = inner - self.call_count = 0 - - def hash_file(self, file_path): - self.call_count += 1 - return self.inner.hash_file(file_path) - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture -def dir_with_files(tmp_path): - """A directory containing three text files.""" - d = tmp_path / "mydir" - d.mkdir() - (d / "a.txt").write_bytes(b"content_a") - (d / "b.txt").write_bytes(b"content_b") - (d / "c.txt").write_bytes(b"content_c") - return d - - -@pytest.fixture() -def restore_default_handlers(): - """Restore both FileHandler and DirectoryHandler after each test.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.directory_type import Directory - from orcapod.extension_types.file_type import File - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - original_file_handler = registry.get_handler_for_type(File) - original_dir_handler = registry.get_handler_for_type(Directory) - yield - registry.register(File, original_file_handler) - registry.register(Directory, original_dir_handler) - - -# --------------------------------------------------------------------------- -# Cache write / hit / invalidation -# --------------------------------------------------------------------------- - - -class TestDirectoryHashCaching: - def test_cache_write_on_first_hash(self, dir_with_files): - """First hash: underlying FileHasher called once per file; results stored.""" - counting = CountingFileHasher(FileHasher()) - cacher = InMemoryHashCacher() - cached = CachedFileHasher(file_hasher=counting, cacher=cacher) - hasher = BasicDirectoryHasher(file_hasher=cached) - - result = hasher.hash_directory(dir_with_files) - - assert isinstance(result, ContentHash) - assert result.method == "merkle_sha256" - assert counting.call_count == 3 # one per file; none cached yet - - def test_cache_hit_on_second_hash(self, dir_with_files): - """Second hash of the same directory returns all results from cache.""" - counting = CountingFileHasher(FileHasher()) - cacher = InMemoryHashCacher() - cached = CachedFileHasher(file_hasher=counting, cacher=cacher) - hasher = BasicDirectoryHasher(file_hasher=cached) - - h1 = hasher.hash_directory(dir_with_files) - count_after_first = counting.call_count # 3 calls for 3 files - - h2 = hasher.hash_directory(dir_with_files) - - assert h1 == h2 # same directory content → same hash - assert counting.call_count == count_after_first # zero new calls; all from cache - - def test_cache_invalidated_on_file_change(self, dir_with_files): - """Modifying a file (mtime_ns/size change) causes a cache miss for that file.""" - counting = CountingFileHasher(FileHasher()) - cacher = InMemoryHashCacher() - cached = CachedFileHasher(file_hasher=counting, cacher=cacher) - hasher = BasicDirectoryHasher(file_hasher=cached) - - hasher.hash_directory(dir_with_files) - count_after_first = counting.call_count # 3 - - # Write different content → changes both mtime_ns and size → cache miss - (dir_with_files / "b.txt").write_bytes(b"modified_content_with_different_length") - - hasher.hash_directory(dir_with_files) - - # Only b.txt triggered a new hash call; a.txt and c.txt came from cache - assert counting.call_count == count_after_first + 1 - - def test_hash_value_unchanged_vs_uncached(self, dir_with_files): - """Cached and uncached hashing produce identical ContentHash values.""" - plain = BasicDirectoryHasher(file_hasher=FileHasher()) - cached = BasicDirectoryHasher( - file_hasher=CachedFileHasher( - file_hasher=FileHasher(), - cacher=InMemoryHashCacher(), - ) - ) - - assert plain.hash_directory(dir_with_files) == cached.hash_directory(dir_with_files) - - -# --------------------------------------------------------------------------- -# enable_file_hash_caching() wires DirectoryHandler -# --------------------------------------------------------------------------- - - -class TestEnableFileHashCachingWiresDirectory: - def test_directory_handler_uses_cached_file_hasher( - self, restore_default_handlers, tmp_path - ): - """After enable_file_hash_caching(), DirectoryHandler uses a CachedFileHasher.""" - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.hashing.file_hashers import CachedFileHasher - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - dir_handler = registry.get_handler_for_type(Directory) - assert isinstance(dir_handler.directory_hasher.file_hasher, CachedFileHasher) - - def test_shared_cache_instance(self, restore_default_handlers, tmp_path): - """FileHandler and DirectoryHandler share the same CachedFileHasher instance.""" - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - - file_hasher = registry.get_handler_for_type(File).file_hasher - dir_file_hasher = registry.get_handler_for_type(Directory).directory_hasher.file_hasher - - assert file_hasher is dir_file_hasher # exact same object -``` - -- [ ] **Step 1.2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_directory_hash_caching.py -v 2>&1 | head -40 -``` - -Expected: `TypeError: BasicDirectoryHasher.__init__() got an unexpected keyword argument 'file_hasher'` (or similar). Tests fail because `file_hasher` is not yet accepted. - ---- - -## Task 2: Fix `BasicDirectoryHasher` — required `file_hasher` parameter - -**Files:** -- Modify: `src/orcapod/hashing/directory_hashers.py` - -- [ ] **Step 2.1: Update `directory_hashers.py`** - -Replace the entire file content with: - -```python -"""BasicDirectoryHasher — recursive Merkle tree hashing for directory trees.""" - -from __future__ import annotations - -import fnmatch -import hashlib -import logging -import os -from collections.abc import Callable - -from upath import UPath - -from orcapod.protocols.hashing_protocols import FileContentHasherProtocol -from orcapod.types import ContentHash, PathLike - -logger = logging.getLogger(__name__) - - -def _compile_ignore( - ignore: Callable[[UPath], bool] | list[str] | None, -) -> Callable[[UPath], bool] | None: - """Convert an ignore spec to a single callable filter. - - Args: - ignore: ``None`` (no filtering), a list of ``fnmatch`` glob patterns matched - against entry names, or a callable ``(UPath) -> bool``. - - Returns: - A callable ``(UPath) -> bool`` returning ``True`` to exclude an entry, or - ``None`` if no filtering is needed. - """ - if ignore is None: - return None - if callable(ignore): - return ignore - patterns = list(ignore) - - def _glob_filter(entry: UPath) -> bool: - return any(fnmatch.fnmatch(entry.name, pat) for pat in patterns) - - return _glob_filter - - -def _hash_dir( - path: UPath, - filter_fn: Callable[[UPath], bool] | None, - algorithm: str, - file_hasher: FileContentHasherProtocol, -) -> bytes: - """Recursively compute the Merkle hash of a directory. - - Args: - path: The directory to hash. - filter_fn: Optional filter callable; return ``True`` to exclude an entry. - algorithm: Hash algorithm name used for structural (entry and node) hashing. - file_hasher: Hasher used to compute ``ContentHash`` for each file leaf. - - Returns: - The raw digest bytes for this directory node (length depends on the algorithm). - """ - entries: list[tuple[bytes, bytes]] = [] - - for child in path.iterdir(): - if filter_fn is not None and filter_fn(child): - continue - - name_bytes = child.name.encode("utf-8") - - if child.is_symlink(): - # Hash-as-symlink: record the link target, never dereference. - target = os.readlink(child) - entry_bytes = b"symlink\x00" + name_bytes + b"\x00" + target.encode("utf-8") - elif child.is_file(): - file_hash = file_hasher.hash_file(child) - entry_bytes = b"file\x00" + name_bytes + b"\x00" + file_hash.digest - elif child.is_dir(): - subdir_digest = _hash_dir(child, filter_fn, algorithm, file_hasher) - entry_bytes = b"dir\x00" + name_bytes + b"\x00" + subdir_digest - else: - # Special file (socket, device node, named pipe) — skip silently. - logger.debug("BasicDirectoryHasher: skipping special file %s", child) - continue - - entry_digest = hashlib.new(algorithm, entry_bytes).digest() - entries.append((name_bytes, entry_digest)) - - # Sort byte-wise by name — locale-independent, deterministic. - entries.sort(key=lambda x: x[0]) - - h = hashlib.new(algorithm) - h.update(b"directory\x00") - for _, entry_digest in entries: - h.update(entry_digest) - return h.digest() - - -class BasicDirectoryHasher: - """Recursive Merkle tree hasher for directory trees. - - Computes a stable content hash of a directory tree using a recursive Merkle scheme: - file leaves hash their content via the injected ``file_hasher``, subdirectory nodes - hash their sorted children, and the root hash propagates the entire tree. Symlinks - are recorded as ``(symlink, target)`` without dereferencing — cycle-safe and - deterministic. - - Pass a ``CachedFileHasher`` as ``file_hasher`` to avoid re-reading unchanged files - on repeated calls (e.g. across pipeline runs). - - Args: - file_hasher: Hasher used to compute the ``ContentHash`` for each file leaf in - the tree. Any object with a ``hash_file(path) -> ContentHash`` method. - algorithm: Hash algorithm for structural (Merkle entry and node) hashing. - Defaults to ``"sha256"``. - buffer_size: Retained for context round-trip use. File content buffer sizing - is owned by ``file_hasher``. Defaults to 65536. - - Example: - >>> from orcapod.hashing.file_hashers import FileHasher - >>> hasher = BasicDirectoryHasher(file_hasher=FileHasher()) - >>> result = hasher.hash_directory("/tmp/mydir") - >>> result.method - 'merkle_sha256' - """ - - def __init__( - self, - file_hasher: FileContentHasherProtocol, - algorithm: str = "sha256", - buffer_size: int = 65536, - ) -> None: - self.file_hasher = file_hasher - self.algorithm = algorithm - self.buffer_size = buffer_size - - def hash_directory( - self, - directory_path: PathLike, - ignore: Callable[[UPath], bool] | list[str] | None = None, - ) -> ContentHash: - """Compute the recursive Merkle hash of a directory tree. - - Args: - directory_path: Path to the directory to hash. - ignore: Optional filter. A callable ``(UPath) -> bool`` returning ``True`` - to exclude an entry, or a list of glob patterns matched against entry - names via ``fnmatch``. Applied at every level of recursion. - - Returns: - A ``ContentHash`` with ``method="merkle_{algorithm}"``. - - Raises: - FileNotFoundError: If ``directory_path`` does not exist. - NotADirectoryError: If ``directory_path`` is not a directory. - PermissionError: If the directory is not traversable. - """ - path = UPath(directory_path) - filter_fn = _compile_ignore(ignore) - digest = _hash_dir(path, filter_fn, self.algorithm, self.file_hasher) - return ContentHash(method=f"merkle_{self.algorithm}", digest=digest) -``` - -- [ ] **Step 2.2: Update `register_builtin_python_type_handlers` fallback** - -In `src/orcapod/hashing/semantic_hashing/builtin_handlers.py`, find the `if directory_hasher is None:` block (currently around line 456–458) and replace it: - -```python - if directory_hasher is None: - from orcapod.hashing.directory_hashers import BasicDirectoryHasher - directory_hasher = BasicDirectoryHasher( - file_hasher=file_hasher, - algorithm="sha256", - ) -``` - -Note: `file_hasher` is resolved in the block immediately above this one, so it is always a concrete `FileContentHasherProtocol` by this point. - -- [ ] **Step 2.3: Run the new caching tests — they should now pass** - -```bash -uv run pytest tests/test_hashing/test_directory_hash_caching.py::TestDirectoryHashCaching -v -``` - -Expected: 4 tests PASS (the `TestEnableFileHashCachingWiresDirectory` class will still fail — that's fine for now). - -- [ ] **Step 2.4: Run the full test suite to see what broke** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -30 -``` - -Expected: failures in `test_directory_handler.py` — `BasicDirectoryHasher()` calls missing the required `file_hasher` argument. - ---- - -## Task 3: Fix existing `test_directory_handler.py` call sites - -**Files:** -- Modify: `tests/test_hashing/test_directory_handler.py` - -- [ ] **Step 3.1: Add `FileHasher` import and update `TestBasicDirectoryHasher`** - -At the top of `tests/test_hashing/test_directory_handler.py`, add to the existing imports: - -```python -from orcapod.hashing.file_hashers import FileHasher -``` - -Then replace every `BasicDirectoryHasher()` call (no args) in `TestBasicDirectoryHasher` with `BasicDirectoryHasher(file_hasher=FileHasher())`. There are approximately 15 such calls; every test method that reads `hasher = BasicDirectoryHasher()` becomes `hasher = BasicDirectoryHasher(file_hasher=FileHasher())`. The affected methods are: - -- `test_empty_directory_returns_content_hash` -- `test_empty_directory_hash_is_stable` (2 uses of `hasher =`) -- `test_identical_content_same_hash` -- `test_different_content_different_hash` -- `test_single_byte_change_in_nested_file_changes_hash` -- `test_adding_file_changes_hash` -- `test_removing_file_changes_hash` -- `test_hidden_files_included_by_default` -- `test_ignore_glob_excludes_matching_files` -- `test_ignore_callable_excludes_entries` -- `test_symlink_recorded_not_followed` -- `test_symlink_cycle_safe` -- `test_large_tree_smoke_test` -- `test_rename_changes_hash` -- `test_ignore_applied_recursively` -- `test_special_files_skipped` - -Also update the `TestDirectoryHandler.handler` fixture: - -```python -@pytest.fixture -def handler(self): - return DirectoryHandler(BasicDirectoryHasher(file_hasher=FileHasher())) -``` - -- [ ] **Step 3.2: Run the directory handler tests** - -```bash -uv run pytest tests/test_hashing/test_directory_handler.py -v -``` - -Expected: all tests PASS. - -- [ ] **Step 3.3: Run the full test suite** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: all tests pass except `TestEnableFileHashCachingWiresDirectory` (addressed in Task 5). - -- [ ] **Step 3.4: Commit** - -```bash -git add src/orcapod/hashing/directory_hashers.py \ - src/orcapod/hashing/semantic_hashing/builtin_handlers.py \ - tests/test_hashing/test_directory_hash_caching.py \ - tests/test_hashing/test_directory_handler.py -git commit -m "feat(hashing): require file_hasher in BasicDirectoryHasher; wire per-file cache (ITL-511)" -``` - ---- - -## Task 4: Wire `file_hasher` reference in `v0.1.json` - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 4.1: Update `directory_hasher` config in `v0.1.json`** - -Find the `"directory_hasher"` key (currently around line 11–16) and replace it: - -```json -"directory_hasher": { - "_class": "orcapod.hashing.directory_hashers.BasicDirectoryHasher", - "_config": { - "algorithm": "sha256", - "file_hasher": {"_ref": "file_hasher"} - } -}, -``` - -- [ ] **Step 4.2: Add a changelog entry** - -In the `"metadata"` → `"changelog"` array, append at the end: - -```json -"Wired file_hasher reference into BasicDirectoryHasher so op.Directory hashing uses the same FileContentHasherProtocol as op.File hashing; required file_hasher constructor arg enforces explicit injection (ITL-511)" -``` - -- [ ] **Step 4.3: Run the full test suite to verify the JSON change is valid** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: same results as after Task 3 (JSON is parsed at context load time; the `_ref` mechanism is already used for `FileHandler`). - -- [ ] **Step 4.4: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json -git commit -m "feat(contexts): wire file_hasher ref into BasicDirectoryHasher in v0.1.json (ITL-511)" -``` - ---- - -## Task 5: Extend `enable_file_hash_caching()` + update `test_hash_cachers.py` - -**Files:** -- Modify: `src/orcapod/contexts/__init__.py` -- Modify: `tests/test_hashing/test_hash_cachers.py` - -- [ ] **Step 5.1: Write failing tests for the directory-patching behaviour** - -In `tests/test_hashing/test_hash_cachers.py`, update the `restore_default_file_handler` fixture to also restore `DirectoryHandler`, then add two new test methods to `TestEnableFileHashCaching`: - -Replace the existing `restore_default_file_handler` fixture with: - -```python -@pytest.fixture() -def restore_default_file_handler(): - """Restore FileHandler and DirectoryHandler after each test.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.directory_type import Directory - from orcapod.extension_types.file_type import File - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - original_file_handler = registry.get_handler_for_type(File) - original_dir_handler = registry.get_handler_for_type(Directory) - yield - registry.register(File, original_file_handler) - registry.register(Directory, original_dir_handler) -``` - -Then add these two methods to `TestEnableFileHashCaching`: - -```python - def test_registers_cached_file_hasher_in_directory_handler( - self, restore_default_file_handler, tmp_path - ): - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.directory_type import Directory - from orcapod.hashing.file_hashers import CachedFileHasher - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - dir_handler = registry.get_handler_for_type(Directory) - assert isinstance(dir_handler.directory_hasher.file_hasher, CachedFileHasher) - - def test_file_and_directory_handlers_share_same_cached_hasher( - self, restore_default_file_handler, tmp_path - ): - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.directory_type import Directory - from orcapod.extension_types.file_type import File - - enable_file_hash_caching(db_path=tmp_path / "cache.db") - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - - file_hasher = registry.get_handler_for_type(File).file_hasher - dir_file_hasher = registry.get_handler_for_type(Directory).directory_hasher.file_hasher - - assert file_hasher is dir_file_hasher -``` - -- [ ] **Step 5.2: Run the new tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching::test_registers_cached_file_hasher_in_directory_handler \ - tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching::test_file_and_directory_handlers_share_same_cached_hasher \ - tests/test_hashing/test_directory_hash_caching.py::TestEnableFileHashCachingWiresDirectory \ - -v -``` - -Expected: all FAIL — `enable_file_hash_caching()` does not yet patch `DirectoryHandler`. - -- [ ] **Step 5.3: Extend `enable_file_hash_caching()` in `src/orcapod/contexts/__init__.py`** - -Replace the tail of `enable_file_hash_caching()` — specifically the final `registry.register(File, ...)` call and everything after it — with the following. This extracts the `CachedFileHasher` into a named variable so it can be shared with `DirectoryHandler`: - -```python - cached_file_hasher = CachedFileHasher( - file_hasher=base_hasher, - cacher=SqliteHashCacher(db_path), - ) - - registry.register( - File, - FileHandler(cached_file_hasher), - ) - - from orcapod.extension_types.directory_type import Directory - from orcapod.hashing.directory_hashers import BasicDirectoryHasher - from orcapod.hashing.semantic_hashing.builtin_handlers import DirectoryHandler - - existing_dir_handler = registry.get_handler_for_type(Directory) - if existing_dir_handler is None: - raise RuntimeError( - "enable_file_hash_caching(): no DirectoryHandler registered for " - "orcapod.Directory in the default context. This should not happen " - "with the standard v0.1 context." - ) - - existing_dir_hasher = existing_dir_handler.directory_hasher - registry.register( - Directory, - DirectoryHandler( - BasicDirectoryHasher( - file_hasher=cached_file_hasher, - algorithm=existing_dir_hasher.algorithm, - buffer_size=existing_dir_hasher.buffer_size, - ) - ), - ) -``` - -Both `FileHandler` and `DirectoryHandler` receive the exact same `cached_file_hasher` instance — sharing the underlying `SqliteHashCacher`. - -- [ ] **Step 5.4: Run all caching tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py \ - tests/test_hashing/test_directory_hash_caching.py \ - -v -``` - -Expected: all tests PASS. - -- [ ] **Step 5.5: Run the full test suite** - -```bash -uv run pytest tests/ -q 2>&1 | tail -10 -``` - -Expected: all tests PASS. - -- [ ] **Step 5.6: Commit** - -```bash -git add src/orcapod/contexts/__init__.py \ - tests/test_hashing/test_hash_cachers.py -git commit -m "feat(contexts): extend enable_file_hash_caching to also patch DirectoryHandler (ITL-511)" -``` - ---- - -## Task 6: User-facing documentation - -**Files:** -- Create: `docs/concepts/file-hash-caching.md` - -- [ ] **Step 6.1: Create the documentation file** - -Create `docs/concepts/file-hash-caching.md`: - -```markdown -# File Hash Caching - -Orcapod can cache per-file content hashes so that large files are only read -once per modification. This is especially useful in pipelines that repeatedly -hash the same large files (e.g. electrophysiology recordings in spike-sort -pipelines). - -## How it works - -`FileHasher` reads file bytes and returns a `ContentHash`. `CachedFileHasher` -wraps any `FileHasher` with a cache backend (`CacherProtocol`): - -```python -from orcapod.hashing.file_hashers import CachedFileHasher, FileHasher -from orcapod.hashing.hash_cachers import SqliteHashCacher - -cacher = SqliteHashCacher() # SQLite-backed cache -cached_hasher = CachedFileHasher(FileHasher(), cacher) -``` - -On each `hash_file()` call the cache is checked first using the key -`(absolute_path, mtime_ns, size)`. A hit returns the stored `ContentHash` -immediately with no disk I/O. A miss reads the file, stores the result, and -returns it. Changing a file's content updates its `mtime_ns` and/or `size`, -which automatically produces a new cache key and invalidates the old entry. - -## Activating caching - -Call `enable_file_hash_caching()` **once at application startup**, before any -file or directory hashing occurs: - -```python -from orcapod.contexts import enable_file_hash_caching - -enable_file_hash_caching() -# or with an explicit database path: -enable_file_hash_caching(db_path="/data/.orcapod/cache.db") -``` - -This wires a `CachedFileHasher` (backed by `SqliteHashCacher`) into the -default context's `FileHandler` and `DirectoryHandler`. After this call, all -`op.File` and `op.Directory` hashing in the default context consults the -cache. - -## `op.Directory` hashing - -`op.Directory` hashing uses a recursive Merkle tree. Each file leaf is hashed -via the injected `file_hasher`. Once `enable_file_hash_caching()` is called, -the same `CachedFileHasher` instance is shared between `FileHandler` and -`DirectoryHandler` — so a file cached via a direct `op.File` hash is also a -cache hit during subsequent directory traversal, and vice versa. - -No additional configuration is required. - -## Cache storage - -| Setting | Value | -|---------|-------| -| Default path | `~/.orcapod/file_hash_cache.db` | -| Env var override | `ORCAPOD_HASH_CACHE_DB=/path/to/cache.db` | -| Custom path | `enable_file_hash_caching(db_path="/path/to/cache.db")` | - -The SQLite database uses WAL mode (safe for single-writer / multi-reader -access) and thread-local connections. - -### Inspecting the cache - -```bash -sqlite3 ~/.orcapod/file_hash_cache.db \ - "SELECT path, mtime_ns, size, cached_at FROM file_hash_cache LIMIT 20;" -``` - -### Clearing the cache - -```python -from orcapod.hashing.hash_cachers import SqliteHashCacher - -SqliteHashCacher().clear() # clears all entries, keeps the database file -``` - -Or delete the file entirely: - -```bash -rm ~/.orcapod/file_hash_cache.db -``` - -## When caching helps - -- **Large files rehashed across pipeline runs** — a 500 MB recording read once - per modification instead of once per pipeline run. -- **Directories with few changes** — only modified files are re-read; unchanged - files are served from cache. - -## When caching doesn't help - -- **Small files** — the SQLite lookup overhead can exceed the time saved. -- **Files that change on every run** — every run is a cache miss; the entry is - written but never read back. -- **First (cold) run** — all files are hashed from scratch regardless. -``` - -- [ ] **Step 6.2: Run the full test suite one final time** - -```bash -uv run pytest tests/ -q 2>&1 | tail -10 -``` - -Expected: all tests PASS. - -- [ ] **Step 6.3: Commit** - -```bash -git add docs/concepts/file-hash-caching.md -git commit -m "docs(concepts): add file-hash-caching activation guide (ITL-511)" -``` - ---- - -## Final check - -- [ ] Run the complete test suite once more and confirm zero failures: - -```bash -uv run pytest tests/ -q -``` - -- [ ] Verify the branch is ready for a PR: - -```bash -git log --oneline main..HEAD -``` - -Expected output (4 commits): -``` - docs(concepts): add file-hash-caching activation guide (ITL-511) - feat(contexts): extend enable_file_hash_caching to also patch DirectoryHandler (ITL-511) - feat(contexts): wire file_hasher ref into BasicDirectoryHasher in v0.1.json (ITL-511) - feat(hashing): require file_hasher in BasicDirectoryHasher; wire per-file cache (ITL-511) -``` diff --git a/superpowers/plans/2026-07-09-itl-473-orcapod-extension-spikeinterface.md b/superpowers/plans/2026-07-09-itl-473-orcapod-extension-spikeinterface.md deleted file mode 100644 index e7706042c..000000000 --- a/superpowers/plans/2026-07-09-itl-473-orcapod-extension-spikeinterface.md +++ /dev/null @@ -1,1592 +0,0 @@ -# ITL-473: orcapod-extension-spikeinterface Split Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Extract all SpikeInterface code from `orcapod-python` into a new standalone pip package `orcapod-extension-spikeinterface`, and introduce a normalized `OrcapodExtension` protocol + `op.register_extension()` API in core orcapod. - -**Architecture:** Phase 1 adds `OrcapodExtension`/`register_extension` to orcapod-python and strips all SI wiring. Phase 2 creates the `nauticalab/orcapod-extension-spikeinterface` GitHub repo with the moved code, a `SpikeInterfaceExtension` singleton, and mirrored CI. Both repos are worked on in the same agent session from the working directory `/home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/`. - -**Tech Stack:** Python 3.11+, uv, hatchling + hatch-vcs, pytest, GitHub Actions (OIDC Trusted Publishing), Linear release action. - ---- - -## File Map - -### orcapod-python (modify/delete) - -| Action | Path | -|--------|------| -| Create | `src/orcapod/extensions.py` | -| Modify | `src/orcapod/__init__.py` | -| Modify | `src/orcapod/extension_types/__init__.py` | -| Modify | `src/orcapod/contexts/data/v0.1.json` | -| Modify | `pyproject.toml` | -| Modify | `.github/workflows/_license-check.yml` | -| Delete | `src/orcapod/extension_types/spikeinterface_types.py` | -| Delete | `tests/test_extension_types/test_spikeinterface_types.py` | -| Create | `tests/test_extensions.py` | -| Git-move | `superpowers/specs/2026-07-01-spikeinterface-baserecording-design.md` | -| Git-move | `superpowers/specs/2026-07-02-si-sorting-logical-type-design.md` | -| Git-move | `superpowers/specs/2026-07-02-spikeinterface-motion-design.md` | -| Git-move | `superpowers/specs/2026-07-03-sorting-analyzer-logical-type-design.md` | -| Regenerate | `uv.lock` | - -### orcapod-extension-spikeinterface (create new repo) - -| Action | Path | -|--------|------| -| Create | `src/orcapod_extension_spikeinterface/__init__.py` | -| Create | `src/orcapod_extension_spikeinterface/_spikeinterface_types.py` | -| Create | `tests/__init__.py` | -| Create | `tests/test_spikeinterface_types.py` | -| Create | `pyproject.toml` | -| Create | `pytest.ini` | -| Create | `.pre-commit-config.yaml` | -| Create | `.github/workflows/run-tests.yml` | -| Create | `.github/workflows/_license-check.yml` | -| Create | `.github/workflows/release.yml` | -| Create | `.github/workflows/release-sync.yml` | -| Create | `CHANGELOG.md` | -| Create | `CLAUDE.md` | -| Create | `README.md` | -| Create | `superpowers/specs/` (four moved SI design specs) | - ---- - -## Phase 1: orcapod-python Changes - ---- - -### Task 1: Create feature branch - -- [ ] **Step 1: Check out the feature branch** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python -git checkout -b eywalker/itl-473-split-spikeinterface-extension-into-separate-orcapod -git branch --show-current -``` - -Expected output: `eywalker/itl-473-split-spikeinterface-extension-into-separate-orcapod` - ---- - -### Task 2: Add `OrcapodExtension` protocol and `register_extension()` - -**Files:** -- Create: `src/orcapod/extensions.py` -- Create: `tests/test_extensions.py` -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_extensions.py`: - -```python -"""Tests for OrcapodExtension protocol and register_extension() (ITL-473).""" - -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest - -from orcapod.extensions import OrcapodExtension, register_extension - - -class _MockExtension: - """Minimal concrete implementation of OrcapodExtension for testing.""" - - name = "mock" - - def register(self, context) -> None: - pass - - -def test_mock_extension_satisfies_protocol(): - """A class with name and register() satisfies OrcapodExtension at runtime.""" - ext = _MockExtension() - assert isinstance(ext, OrcapodExtension) - - -def test_register_extension_passes_explicit_context(): - """register_extension passes an explicit context directly to extension.register.""" - ext = MagicMock(spec=OrcapodExtension) - mock_context = MagicMock() - - register_extension(ext, context=mock_context) - - ext.register.assert_called_once_with(mock_context) - - -def test_register_extension_resolves_default_context_when_none(): - """register_extension calls get_default_context() and passes result when context=None.""" - ext = MagicMock(spec=OrcapodExtension) - mock_context = MagicMock() - - with patch("orcapod.extensions.get_default_context", return_value=mock_context) as mock_get: - register_extension(ext, context=None) - - mock_get.assert_called_once() - ext.register.assert_called_once_with(mock_context) - - -def test_register_extension_context_defaults_to_none(): - """register_extension resolves default context when called without context kwarg.""" - ext = MagicMock(spec=OrcapodExtension) - mock_context = MagicMock() - - with patch("orcapod.extensions.get_default_context", return_value=mock_context): - register_extension(ext) # no context arg - - ext.register.assert_called_once_with(mock_context) - - -def test_register_extension_does_not_call_get_default_context_when_context_provided(): - """register_extension never calls get_default_context when a context is supplied.""" - ext = MagicMock(spec=OrcapodExtension) - mock_context = MagicMock() - - with patch("orcapod.extensions.get_default_context") as mock_get: - register_extension(ext, context=mock_context) - - mock_get.assert_not_called() -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extensions.py -v -``` - -Expected: `ModuleNotFoundError: No module named 'orcapod.extensions'` - -- [ ] **Step 3: Create `src/orcapod/extensions.py`** - -```python -"""Normalized extension registration API for orcapod (ITL-473). - -Third-party extension packages expose a module-level singleton that implements -``OrcapodExtension`` and register it via ``op.register_extension()``. - -Example: - >>> import orcapod as op - >>> from orcapod_extension_spikeinterface import spikeinterface_extension - >>> op.register_extension(spikeinterface_extension) -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol, runtime_checkable - -if TYPE_CHECKING: - from orcapod.contexts import DataContext - - -@runtime_checkable -class OrcapodExtension(Protocol): - """Protocol that all orcapod extension objects must implement. - - Extension packages expose a module-level singleton instance of a class - that implements this protocol and register it via ``op.register_extension()``. - - Example: - >>> import orcapod as op - >>> from orcapod_extension_spikeinterface import spikeinterface_extension - >>> op.register_extension(spikeinterface_extension) - - Attributes: - name: Short identifier used in log messages (e.g. ``"spikeinterface"``). - """ - - name: str - - def register(self, context: DataContext) -> None: - """Register this extension's types into ``context``. - - ``context`` is always a concrete ``DataContext`` — never ``None``. - Context resolution (default vs. explicit) is handled by - ``register_extension`` before this method is called. - - Args: - context: Target ``DataContext`` to register types into. - """ - ... - - -def register_extension( - extension: OrcapodExtension, - context: DataContext | None = None, -) -> None: - """Register an extension into a data context. - - Resolves ``context`` to the default context when ``None``, then delegates - to ``extension.register(context)`` with the concrete context. Extensions - never receive ``None`` for ``context``. - - Args: - extension: An object implementing ``OrcapodExtension``. - context: Target ``DataContext``. Resolves to the default context - if ``None``. - - Example: - >>> import orcapod as op - >>> from orcapod_extension_spikeinterface import spikeinterface_extension - >>> op.register_extension(spikeinterface_extension) - >>> # or against a specific context: - >>> op.register_extension(spikeinterface_extension, context=my_context) - """ - from orcapod.contexts import get_default_context - - if context is None: - context = get_default_context() - extension.register(context) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extensions.py -v -``` - -Expected: 5 tests PASSED. - -- [ ] **Step 5: Export from `src/orcapod/__init__.py`** - -Add to `src/orcapod/__init__.py` after the existing imports (after line 35, before `__all__`): - -```python -from .extensions import OrcapodExtension, register_extension -``` - -And add to `__all__`: - -```python - "OrcapodExtension", - "register_extension", -``` - -The final `__init__.py` should look like: - -```python -from .config import ( - DEFAULT_CONFIG, - DisplayConfig, - HashingConfig, - OrcapodConfig, - load_config, -) -from .core.function_pod import ( - FunctionPod, - function_pod, -) -from .core.nodes.source_node import SourceNode -from .pipeline import Pipeline, PipelineJob -# Subpackage re-exports for clean public API -from . import databases # noqa: F401 -from . import nodes # noqa: F401 -from . import operators # noqa: F401 -from . import sources # noqa: F401 -from . import streams # noqa: F401 -from . import types # noqa: F401 - -# Stable type aliases — preferred over importing directly from pathlib/upath/uuid. -# -# These aliases are the recommended way to reference these types in orcapod user code. -# Even if an upstream library is renamed or restructured, these symbols remain stable -# at ``orcapod.Path``, ``orcapod.UPath``, ``orcapod.UUID``, ``orcapod.File``, and -# ``orcapod.Directory``. Their Arrow extension types are registered under the -# ``orcapod.*`` namespace (``"orcapod.path"``, ``"orcapod.upath"``, ``"orcapod.uuid"``, -# ``"orcapod.file"``, ``"orcapod.directory"``), so on-disk identity is also decoupled -# from upstream module paths. -from pathlib import Path -from upath import UPath -from uuid import UUID -from orcapod.extension_types.file_type import File -from orcapod.extension_types.directory_type import Directory - -# Extension registration API (ITL-473) -from .extensions import OrcapodExtension, register_extension - -__all__ = [ - "DEFAULT_CONFIG", - "DisplayConfig", - "HashingConfig", - "OrcapodConfig", - "load_config", - "FunctionPod", - "function_pod", - "Pipeline", - "PipelineJob", - "SourceNode", - "databases", - "nodes", - "operators", - "sources", - "streams", - "types", - # Stable type aliases - "Directory", - "File", - "Path", - "UPath", - "UUID", - # Extension registration API (ITL-473) - "OrcapodExtension", - "register_extension", -] -``` - -- [ ] **Step 6: Verify the export works** - -```bash -uv run python -c "import orcapod as op; print(op.register_extension, op.OrcapodExtension)" -``` - -Expected: prints ` ` - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/extensions.py src/orcapod/__init__.py tests/test_extensions.py -git commit -m "feat(extensions): add OrcapodExtension protocol and register_extension() (ITL-473)" -``` - ---- - -### Task 3: Strip SI block from `extension_types/__init__.py` - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` - -- [ ] **Step 1: Remove the try/except SI block** - -Open `src/orcapod/extension_types/__init__.py`. Remove this entire block (currently near the bottom of the imports, after `LogicalPandasDataFrame`): - -```python -# ITL-459, ITL-468, ITL-470, ITL-469 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) -try: - from .spikeinterface_types import ( - LogicalSIRecording, - LogicalSISorting, - LogicalSIMotion, - LogicalSISortingAnalyzer, - SIMotionHandler, - SISortingAnalyzerHandler, - register_spikeinterface_types, - ) - _SI_AVAILABLE = True -except ImportError: - _SI_AVAILABLE = False -``` - -Also remove the conditional `__all__` extension at the bottom: - -```python - # ITL-459, ITL-468, ITL-470, ITL-469 (conditional — only present when spikeinterface is installed) - *( - [ - "LogicalSIRecording", - "LogicalSISorting", - "LogicalSIMotion", - "LogicalSISortingAnalyzer", - "SIMotionHandler", - "SISortingAnalyzerHandler", - "register_spikeinterface_types", - ] if _SI_AVAILABLE else [] - ), -``` - -- [ ] **Step 2: Verify the module imports cleanly** - -```bash -uv run python -c "from orcapod.extension_types import LogicalTypeRegistry; print('ok')" -``` - -Expected: `ok` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py -git commit -m "refactor(extension_types): remove SI conditional re-export block (ITL-473)" -``` - ---- - -### Task 4: Strip SI entries from `v0.1.json` - -**Files:** -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Remove the four SI logical_type entries** - -In `src/orcapod/contexts/data/v0.1.json`, under `type_converter._config.logical_type_registry._config.logical_types`, remove these four objects (they have `"_optional": true`): - -```json -{ - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSIRecording", - "_config": {}, - "_optional": true -}, -{ - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSISorting", - "_config": {}, - "_optional": true -}, -{ - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSIMotion", - "_config": {}, - "_optional": true -}, -{ - "_class": "orcapod.extension_types.spikeinterface_types.LogicalSISortingAnalyzer", - "_config": {}, - "_optional": true -}, -``` - -- [ ] **Step 2: Remove the four SI handler entries** - -In the same file, under `python_type_handler_registry._config.handlers`, remove these four pairs (they have `"_optional": true`): - -```json -[{"_type": "spikeinterface.core.BaseRecording", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SIRecordingHandler", "_config": {}, "_optional": true}], -[{"_type": "spikeinterface.core.BaseSorting", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SISortingHandler", "_config": {}, "_optional": true}], -[{"_type": "spikeinterface.core.motion.Motion", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SIMotionHandler", "_config": {}, "_optional": true}], -[{"_type": "spikeinterface.core.SortingAnalyzer", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SISortingAnalyzerHandler", "_config": {}, "_optional": true}], -``` - -- [ ] **Step 3: Verify the JSON is valid and the default context loads** - -```bash -uv run python -c "from orcapod.contexts import get_default_context; ctx = get_default_context(); print('context_key:', ctx.context_key)" -``` - -Expected: `context_key: std:v0.1:default` - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/contexts/data/v0.1.json -git commit -m "refactor(contexts): remove SI _optional entries from v0.1.json (ITL-473)" -``` - ---- - -### Task 5: Remove spikeinterface from `pyproject.toml` and license workflow - -**Files:** -- Modify: `pyproject.toml` -- Modify: `.github/workflows/_license-check.yml` - -- [ ] **Step 1: Remove spikeinterface from `pyproject.toml`** - -In `pyproject.toml`, make three changes: - -1. Remove the `spikeinterface` optional-dependency line: -```toml -spikeinterface = ["spikeinterface>=0.101"] -``` - -2. Remove `"orcapod[spikeinterface]"` from the `all` extras list: -```toml -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]", "orcapod[spikeinterface]"] -``` -becomes: -```toml -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]"] -``` - -3. Remove `"spikeinterface>=0.101"` from `[dependency-groups] dev`: -```toml - "spikeinterface>=0.101", -``` - -- [ ] **Step 2: Remove `--ignore-packages quantities` from license check** - -In `.github/workflows/_license-check.yml`, the last line of the `pip-licenses` command reads: -```yaml - --ignore-packages quantities -``` - -Remove `--ignore-packages quantities` from that invocation. The final command should end at `--partial-match`. - -- [ ] **Step 3: Commit** - -```bash -git add pyproject.toml .github/workflows/_license-check.yml -git commit -m "chore(deps): remove spikeinterface from orcapod-python deps and license check (ITL-473)" -``` - ---- - -### Task 6: Delete SI source file, test file, and regenerate lock - -**Files:** -- Delete: `src/orcapod/extension_types/spikeinterface_types.py` -- Delete: `tests/test_extension_types/test_spikeinterface_types.py` -- Regenerate: `uv.lock` - -- [ ] **Step 1: Delete the files** - -```bash -git rm src/orcapod/extension_types/spikeinterface_types.py -git rm tests/test_extension_types/test_spikeinterface_types.py -``` - -- [ ] **Step 2: Regenerate `uv.lock`** - -```bash -uv lock -``` - -This may take a minute — uv is resolving dependencies without spikeinterface and its transitive deps (neo, quantities, probeinterface, zarr, numcodecs, fasteners, threadpoolctl, asciitree). The lock file will shrink substantially. - -- [ ] **Step 3: Run the core test suite (without spikeinterface)** - -```bash -uv run pytest -m "not postgres" --tb=short -q -``` - -Expected: All tests pass. No SI-related import errors or skips. The deleted SI test file is gone so no SI tests run. - -- [ ] **Step 4: Commit** - -```bash -git add uv.lock -git commit -m "feat(itl-473): delete spikeinterface_types from core, regenerate lock" -``` - ---- - -### Task 7: Move SI design specs - -**Files:** -- Git-move 4 files from `superpowers/specs/` to a staging location (they'll be put in the new repo in Phase 2) - -- [ ] **Step 1: Move the four SI spec files out of orcapod-python** - -```bash -git mv superpowers/specs/2026-07-01-spikeinterface-baserecording-design.md superpowers/specs/_moved_to_extension_repo_2026-07-01-spikeinterface-baserecording-design.md -git mv superpowers/specs/2026-07-02-si-sorting-logical-type-design.md superpowers/specs/_moved_to_extension_repo_2026-07-02-si-sorting-logical-type-design.md -git mv superpowers/specs/2026-07-02-spikeinterface-motion-design.md superpowers/specs/_moved_to_extension_repo_2026-07-02-spikeinterface-motion-design.md -git mv superpowers/specs/2026-07-03-sorting-analyzer-logical-type-design.md superpowers/specs/_moved_to_extension_repo_2026-07-03-sorting-analyzer-logical-type-design.md -``` - -Note: the `_moved_to_extension_repo_` prefix is temporary — we'll copy the content into the new repo in Phase 2, then delete these from orcapod-python. - -- [ ] **Step 2: Commit** - -```bash -git add superpowers/specs/ -git commit -m "docs(itl-473): mark SI design specs as moved to extension repo" -``` - ---- - -### Task 8: Open orcapod-python PR - -- [ ] **Step 1: Push the feature branch** - -```bash -git push -u origin eywalker/itl-473-split-spikeinterface-extension-into-separate-orcapod -``` - -- [ ] **Step 2: Open the PR** - -```bash -gh pr create \ - --title "feat(itl-473): split spikeinterface into orcapod-extension-spikeinterface" \ - --body "$(cat <<'EOF' -## Summary - -- Adds `OrcapodExtension` protocol and `op.register_extension()` for normalized extension registration (ITL-473) -- Removes all SpikeInterface wiring from core orcapod: `spikeinterface_types.py`, `v0.1.json` entries, `extension_types/__init__.py` re-export, optional deps, and license check exclusion -- Regenerates `uv.lock` without spikeinterface transitive deps -- SI code moves to new `nauticalab/orcapod-extension-spikeinterface` package - -## Test plan -- [ ] Core test suite passes without spikeinterface installed (`uv run pytest -m "not postgres"`) -- [ ] `import orcapod as op; op.register_extension` is accessible -- [ ] `from orcapod.contexts import get_default_context; get_default_context()` works (no SI import errors) -- [ ] License check passes without `--ignore-packages quantities` - -Closes ITL-473 -EOF -)" -``` - -Record the PR URL returned by the command. - ---- - -## Phase 2: Create `orcapod-extension-spikeinterface` Repo - ---- - -### Task 9: Create and clone the new GitHub repository - -- [ ] **Step 1: Create the repo** - -```bash -gh repo create nauticalab/orcapod-extension-spikeinterface \ - --public \ - --description "OrcaPod extension for SpikeInterface types (BaseRecording, BaseSorting, Motion, SortingAnalyzer)" -``` - -- [ ] **Step 2: Clone into the working directory** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0 -gh repo clone nauticalab/orcapod-extension-spikeinterface -cd orcapod-extension-spikeinterface -git config user.name "agent-kurodo[bot]" -git config user.email "268466204+agent-kurodo[bot]@users.noreply.github.com" -``` - -- [ ] **Step 3: Create the directory structure** - -```bash -mkdir -p src/orcapod_extension_spikeinterface -mkdir -p tests -mkdir -p .github/workflows -mkdir -p superpowers/specs -``` - ---- - -### Task 10: Create `pyproject.toml` and boilerplate - -**Files:** -- Create: `pyproject.toml` -- Create: `pytest.ini` -- Create: `.pre-commit-config.yaml` -- Create: `CHANGELOG.md` -- Create: `README.md` -- Create: `CLAUDE.md` - -- [ ] **Step 1: Create `pyproject.toml`** - -```toml -[build-system] -requires = ["hatchling>=1.21.0", "hatch-vcs>=0.4.0"] -build-backend = "hatchling.build" - -[project] -name = "orcapod-extension-spikeinterface" -description = "OrcaPod extension for SpikeInterface types (BaseRecording, BaseSorting, Motion, SortingAnalyzer)" -dynamic = ["version"] -dependencies = [ - "orcapod>=0.1", - "spikeinterface>=0.101", -] -readme = "README.md" -requires-python = ">=3.11.0" -license = { text = "MIT License" } -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", -] - -[project.urls] -Homepage = "https://github.com/nauticalab/orcapod-extension-spikeinterface" - -[tool.hatch.version] -source = "vcs" - -[tool.hatch.build.hooks.vcs] -version-file = "src/orcapod_extension_spikeinterface/_version.py" - -[tool.hatch.build.targets.wheel] -packages = ["src/orcapod_extension_spikeinterface"] - - -[dependency-groups] -dev = [ - "pytest>=8.3.5", - "pytest-cov>=6.1.1", - "ruff>=0.14.4", - "pre-commit>=4.4.0", - "pre-commit-hooks>=6.0.0", - "pip-licenses>=5.0.0", -] - - -[tool.coverage.run] -omit = [] - -[tool.coverage.report] -exclude_also = [ - "if TYPE_CHECKING:", -] -``` - -- [ ] **Step 2: Create `pytest.ini`** - -```ini -[pytest] -testpaths = tests -python_files = test_*.py -python_classes = Test* -python_functions = test_* -addopts = -v -pythonpath = src -``` - -- [ ] **Step 3: Create `.pre-commit-config.yaml`** - -```yaml -repos: - - repo: https://github.com/tsvikas/sync-with-uv - rev: v0.4.0 - hooks: - - id: sync-with-uv - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.4 - hooks: - - id: ruff-format - types_or: [ python, pyi ] - - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: trailing-whitespace - types_or: [ python, pyi ] - - id: end-of-file-fixer - types_or: [ python, pyi ] - - id: check-yaml - - id: check-added-large-files - - id: check-merge-conflict -``` - -- [ ] **Step 4: Create `CHANGELOG.md`** - -```markdown -# Changelog - -## Unreleased - -### Added - -- Initial release: extracted from `orcapod-python` (ITL-473) -- `LogicalSIRecording`, `LogicalSISorting`, `LogicalSIMotion`, `LogicalSISortingAnalyzer` logical types -- `SIRecordingHandler`, `SISortingHandler`, `SIMotionHandler`, `SISortingAnalyzerHandler` semantic hashers -- `SpikeInterfaceExtension` class and `spikeinterface_extension` singleton -- `op.register_extension(spikeinterface_extension)` registration API -``` - -- [ ] **Step 5: Create `README.md`** - -```markdown -# orcapod-extension-spikeinterface - -OrcaPod extension for [SpikeInterface](https://spikeinterface.readthedocs.io/) types. - -Adds native support for `BaseRecording`, `BaseSorting`, `Motion`, and `SortingAnalyzer` -objects as first-class orcapod value types. - -## Installation - -```bash -pip install orcapod-extension-spikeinterface -``` - -## Usage - -```python -import orcapod as op -from orcapod_extension_spikeinterface import spikeinterface_extension - -# Register SI types into the default orcapod context at startup -op.register_extension(spikeinterface_extension) -``` - -After registration, SpikeInterface objects can be used as typed inputs/outputs in -orcapod `FunctionPod` functions. - -## License - -MIT -``` - -- [ ] **Step 6: Create `CLAUDE.md`** - -```markdown -# Claude Code instructions for orcapod-extension-spikeinterface - -## Running commands - -Always run Python commands via `uv run`, e.g.: - -``` -uv run pytest tests/ -uv run python -c "..." -``` - -Never use `python`, `pytest`, or `python3` directly. - -## Superpowers artifacts - -Place all superpowers-related artifacts (design specs, plans, etc.) in the `superpowers/` -directory at the project root. - -- Specs go in `superpowers/specs/` -- Plans go in `superpowers/plans/` - -## Docstrings - -Use [Google style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) -Python docstrings everywhere. Never mix in ReST markup. - -## Git commits - -Always use [Conventional Commits](https://www.conventionalcommits.org/) style: - -``` -(): -``` -``` - -- [ ] **Step 7: Commit boilerplate** - -```bash -git add pyproject.toml pytest.ini .pre-commit-config.yaml CHANGELOG.md README.md CLAUDE.md -git commit -m "chore: initial repo scaffold for orcapod-extension-spikeinterface (ITL-473)" -``` - ---- - -### Task 11: Create `_spikeinterface_types.py` (moved and updated) - -**Files:** -- Create: `src/orcapod_extension_spikeinterface/_spikeinterface_types.py` - -- [ ] **Step 1: Copy the source file from orcapod-python** - -```bash -cp /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python/src/orcapod/extension_types/spikeinterface_types.py \ - src/orcapod_extension_spikeinterface/_spikeinterface_types.py -``` - -- [ ] **Step 2: Update the module docstring** - -At the top of `src/orcapod_extension_spikeinterface/_spikeinterface_types.py`, replace: - -```python -This module requires the optional ``spikeinterface`` extras group: -``pip install orcapod[spikeinterface]`` - -Register SI types into the default orcapod context before using them in -pods: call ``register_spikeinterface_types()`` once at startup. -``` - -with: - -```python -This module is part of the ``orcapod-extension-spikeinterface`` package: -``pip install orcapod-extension-spikeinterface`` - -Register SI types into the default orcapod context before using them in -pods via the normalized extension API:: - - import orcapod as op - from orcapod_extension_spikeinterface import spikeinterface_extension - op.register_extension(spikeinterface_extension) -``` - -- [ ] **Step 3: Update the ImportError message** - -Find and replace the ImportError message in the `try/except` block near the top of the file: - -```python - raise ImportError( - "spikeinterface is not installed. " - "Install it with: pip install orcapod[spikeinterface]" -``` - -Replace with: - -```python - raise ImportError( - "spikeinterface is not installed. " - "Install it with: pip install orcapod-extension-spikeinterface" -``` - -- [ ] **Step 4: Rename `register_spikeinterface_types` → `_register_spikeinterface_types` and update signature** - -Find the function definition at the bottom of the file: - -```python -def register_spikeinterface_types(context: Any = None) -> None: - """Register SpikeInterface LogicalTypes into an orcapod ``DataContext``. - ... - If ``context`` is ``None``, the default context (from - ``orcapod.contexts.get_default_context()``) is used. The function is - idempotent — calling it more than once on the same context is safe. - - Args: - context: A ``DataContext`` instance, or ``None`` to use the default. - - Example: - >>> from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types - >>> register_spikeinterface_types() # no-op if default context already has SI types - """ - if context is None: - from orcapod.contexts import get_default_context - context = get_default_context() -``` - -Replace with: - -```python -def _register_spikeinterface_types(context: "DataContext") -> None: - """Register SpikeInterface LogicalTypes into an orcapod ``DataContext``. - - Registers ``LogicalSIRecording`` / ``SIRecordingHandler`` (ITL-459), - ``LogicalSISorting`` / ``SISortingHandler`` (ITL-468), - ``LogicalSIMotion`` / ``SIMotionHandler`` (ITL-470), and - ``LogicalSISortingAnalyzer`` / ``SISortingAnalyzerHandler`` (ITL-469). - - This is an internal implementation function. Call via - ``op.register_extension(spikeinterface_extension)`` instead. - - The function is idempotent — calling it more than once on the same - context is safe. - - Args: - context: A concrete ``DataContext`` instance. Never ``None`` — - context resolution is handled by ``op.register_extension``. - """ -``` - -Remove the `if context is None:` block entirely (the three lines after the old docstring). - -Also update the `TYPE_CHECKING` guard at the top if `Any` is only used in the old signature — remove `Any` from the import if it's no longer needed. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod_extension_spikeinterface/_spikeinterface_types.py -git commit -m "feat(spikeinterface_types): move and update from orcapod-python (ITL-473)" -``` - ---- - -### Task 12: Create `__init__.py` for the new package - -**Files:** -- Create: `src/orcapod_extension_spikeinterface/__init__.py` - -- [ ] **Step 1: Create the file** - -```python -"""OrcaPod extension for SpikeInterface types (ITL-473). - -Provides ``LogicalSIRecording``, ``LogicalSISorting``, ``LogicalSIMotion``, -``LogicalSISortingAnalyzer`` and their corresponding semantic hashers as -first-class orcapod value types. - -Register via the normalized extension API:: - - import orcapod as op - from orcapod_extension_spikeinterface import spikeinterface_extension - op.register_extension(spikeinterface_extension) -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from ._spikeinterface_types import ( - LogicalSIRecording, - LogicalSISorting, - LogicalSIMotion, - LogicalSISortingAnalyzer, - SIRecordingHandler, - SISortingHandler, - SIMotionHandler, - SISortingAnalyzerHandler, - _register_spikeinterface_types, -) - -if TYPE_CHECKING: - from orcapod.contexts import DataContext - - -class SpikeInterfaceExtension: - """OrcaPod extension object for SpikeInterface types. - - Implements ``orcapod.OrcapodExtension``. The canonical way to use this - extension is via the module-level ``spikeinterface_extension`` singleton:: - - import orcapod as op - from orcapod_extension_spikeinterface import spikeinterface_extension - op.register_extension(spikeinterface_extension) - - Attributes: - name: Extension identifier (``"spikeinterface"``). - """ - - name = "spikeinterface" - - def register(self, context: DataContext) -> None: - """Register all SpikeInterface types into ``context``. - - Registers ``LogicalSIRecording``, ``LogicalSISorting``, - ``LogicalSIMotion``, ``LogicalSISortingAnalyzer`` and their handlers. - Idempotent — safe to call multiple times on the same context. - - Args: - context: Target ``DataContext``. Always a concrete context — - resolution from ``None`` is handled by ``op.register_extension``. - """ - _register_spikeinterface_types(context) - - -#: Module-level singleton — the canonical extension object. -#: Pass to ``op.register_extension()`` to wire SI types into a context. -spikeinterface_extension = SpikeInterfaceExtension() - -__all__ = [ - "SpikeInterfaceExtension", - "spikeinterface_extension", - "LogicalSIRecording", - "LogicalSISorting", - "LogicalSIMotion", - "LogicalSISortingAnalyzer", - "SIRecordingHandler", - "SISortingHandler", - "SIMotionHandler", - "SISortingAnalyzerHandler", -] -``` - -- [ ] **Step 2: Create empty `tests/__init__.py`** - -```bash -touch tests/__init__.py -``` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod_extension_spikeinterface/__init__.py tests/__init__.py -git commit -m "feat: add SpikeInterfaceExtension class and spikeinterface_extension singleton (ITL-473)" -``` - ---- - -### Task 13: Port and update the test file - -**Files:** -- Create: `tests/test_spikeinterface_types.py` - -- [ ] **Step 1: Copy the test file from orcapod-python** - -```bash -cp /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python/tests/test_extension_types/test_spikeinterface_types.py \ - tests/test_spikeinterface_types.py -``` - -- [ ] **Step 2: Update the module-key string in the import-error test** - -In `tests/test_spikeinterface_types.py`, find `test_spikeinterface_not_installed_raises_import_error`. Change: - -```python - si_types_key = "orcapod.extension_types.spikeinterface_types" -``` - -to: - -```python - si_types_key = "orcapod_extension_spikeinterface._spikeinterface_types" -``` - -Also update the expected error message assertion from: - -```python - assert "pip install orcapod[spikeinterface]" in str(exc_info.value) -``` - -to: - -```python - assert "pip install orcapod-extension-spikeinterface" in str(exc_info.value) -``` - -- [ ] **Step 3: Update all `from orcapod.extension_types.spikeinterface_types import` to the new path** - -Replace all occurrences of: - -```python -from orcapod.extension_types.spikeinterface_types import -``` - -with: - -```python -from orcapod_extension_spikeinterface._spikeinterface_types import -``` - -This affects approximately 30 import statements across the test functions. Use a find-and-replace across the whole file. - -- [ ] **Step 4: Update the registration tests to use the new API** - -Find `test_register_spikeinterface_types` and all other `test_register_spikeinterface_types_*` functions. They currently call: - -```python -from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types -... -register_spikeinterface_types() -``` - -Replace the import and call in each registration test with the new API: - -```python -import orcapod as op -from orcapod_extension_spikeinterface import spikeinterface_extension -from orcapod.contexts import get_default_context -... -op.register_extension(spikeinterface_extension) -ctx = get_default_context() -``` - -There are 7 registration tests (lines 322, 354, 535, 565, 587, 791, 833). Update each one. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_spikeinterface_types.py -git commit -m "test: port spikeinterface_types tests to new package (ITL-473)" -``` - ---- - -### Task 14: Create CI workflows - -**Files:** -- Create: `.github/workflows/run-tests.yml` -- Create: `.github/workflows/_license-check.yml` -- Create: `.github/workflows/release.yml` -- Create: `.github/workflows/release-sync.yml` - -- [ ] **Step 1: Create `run-tests.yml`** - -```yaml -name: Run Tests - -on: - push: - branches: [main, dev] - pull_request: - workflow_dispatch: - -jobs: - license-check: - uses: ./.github/workflows/_license-check.yml - - test: - needs: license-check - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.11", "3.12"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: uv sync --locked --all-groups - - - name: Run tests - run: uv run pytest --cov=src --cov-report=term-missing --cov-report=xml - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - dependency-review: - name: Dependency review - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v4 - - - name: Dependency review - uses: actions/dependency-review-action@v4 - with: - deny-licenses: >- - GPL-2.0-only, GPL-2.0-or-later, - GPL-3.0-only, GPL-3.0-or-later, - AGPL-3.0-only, AGPL-3.0-or-later, - LGPL-2.0-only, LGPL-2.0-or-later, - LGPL-2.1-only, LGPL-2.1-or-later, - LGPL-3.0-only, LGPL-3.0-or-later -``` - -- [ ] **Step 2: Create `_license-check.yml`** - -```yaml -name: License check - -on: - workflow_call: - -jobs: - license-check: - name: License check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: uv sync --locked --dev - - - name: Check dependency licenses - run: >- - uv run pip-licenses - --allow-only="MIT;Apache-2.0;Apache 2.0;Apache Software License;BSD-2-Clause;BSD-3-Clause;BSD License;BSD;ISC;LGPL-3.0-only;MPL-2.0;Mozilla Public License;Python-2.0;PSF-2.0;Python Software Foundation License;Unlicense" - --partial-match -``` - -Note: `quantities` is **not** ignored here — it is a legitimate first-class transitive dep of spikeinterface in this package. - -- [ ] **Step 3: Create `release.yml`** - -```yaml -name: Release - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g. 0.1.0 or v0.1.0 — leading v is stripped automatically)' - required: true - type: string - -jobs: - test: - name: Test (Python ${{ matrix.python-version }}) - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: true - matrix: - python-version: ["3.11", "3.12"] - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Install dependencies - run: uv sync --locked --dev --python ${{ matrix.python-version }} - - - name: Run tests - run: uv run --python ${{ matrix.python-version }} pytest --tb=short -q - - license-check: - uses: ./.github/workflows/_license-check.yml - - build: - name: Build distribution - needs: [test, license-check] - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: write - outputs: - version: ${{ steps.normalize.outputs.version }} - steps: - - name: Normalize version - id: normalize - run: | - VERSION="${{ inputs.version }}" - VERSION="${VERSION#v}" - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - - name: Configure git identity - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Create local release tag - run: git tag "v${{ steps.normalize.outputs.version }}" - - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Build wheel and sdist - run: uv build - - - name: Guard against duplicate tag - run: | - if git ls-remote --tags origin "refs/tags/v${{ steps.normalize.outputs.version }}" | grep -q .; then - echo "::error::Tag v${{ steps.normalize.outputs.version }} already exists on origin. Aborting." - exit 1 - fi - - - name: Push release tag - run: git push origin "v${{ steps.normalize.outputs.version }}" - - - name: Upload dist artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: dist - path: dist/ - if-no-files-found: error - - publish-testpypi: - name: Publish → TestPyPI - needs: build - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: - name: testpypi - url: https://test.pypi.org/p/orcapod-extension-spikeinterface - permissions: - id-token: write - steps: - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Download dist artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: dist - path: dist/ - - - name: Publish to TestPyPI - run: uv publish --publish-url https://test.pypi.org/legacy/ dist/* - - publish-pypi: - name: Publish → PyPI - needs: [build, publish-testpypi] - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: - name: pypi - url: https://pypi.org/p/orcapod-extension-spikeinterface - permissions: - id-token: write - contents: write - steps: - - name: Install uv - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 - - - name: Download dist artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: dist - path: dist/ - - - name: Publish to PyPI - run: uv publish dist/* - - - name: Create GitHub Release - uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 - with: - tag_name: "v${{ needs.build.outputs.version }}" - generate_release_notes: true - files: dist/* - - linear-sync: - name: Sync Linear release - needs: build - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - name: Sync Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: sync - version: "v${{ needs.build.outputs.version }}" - - linear-complete: - name: Complete Linear release - needs: [build, publish-pypi, linear-sync] - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - name: Complete Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: complete - version: "v${{ needs.build.outputs.version }}" -``` - -- [ ] **Step 4: Create `release-sync.yml`** - -```yaml -name: Linear release sync - -on: - push: - branches: - - main - -jobs: - sync: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Sync Linear release - uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f # v0 - with: - access_key: ${{ secrets.LINEAR_ACCESS_KEY }} - command: sync -``` - -- [ ] **Step 5: Commit CI workflows** - -```bash -git add .github/workflows/ -git commit -m "ci: add run-tests, license-check, release, release-sync workflows (ITL-473)" -``` - ---- - -### Task 15: Move SI design specs into new repo - -- [ ] **Step 1: Copy the spec files from orcapod-python** - -```bash -cp /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python/superpowers/specs/_moved_to_extension_repo_2026-07-01-spikeinterface-baserecording-design.md \ - superpowers/specs/2026-07-01-spikeinterface-baserecording-design.md -cp /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python/superpowers/specs/_moved_to_extension_repo_2026-07-02-si-sorting-logical-type-design.md \ - superpowers/specs/2026-07-02-si-sorting-logical-type-design.md -cp /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python/superpowers/specs/_moved_to_extension_repo_2026-07-02-spikeinterface-motion-design.md \ - superpowers/specs/2026-07-02-spikeinterface-motion-design.md -cp /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python/superpowers/specs/_moved_to_extension_repo_2026-07-03-sorting-analyzer-logical-type-design.md \ - superpowers/specs/2026-07-03-sorting-analyzer-logical-type-design.md -``` - -- [ ] **Step 2: Remove the staged-for-deletion copies from orcapod-python** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python -git rm superpowers/specs/_moved_to_extension_repo_2026-07-01-spikeinterface-baserecording-design.md -git rm superpowers/specs/_moved_to_extension_repo_2026-07-02-si-sorting-logical-type-design.md -git rm superpowers/specs/_moved_to_extension_repo_2026-07-02-spikeinterface-motion-design.md -git rm superpowers/specs/_moved_to_extension_repo_2026-07-03-sorting-analyzer-logical-type-design.md -git commit -m "docs(itl-473): remove SI design specs moved to extension repo" -``` - -- [ ] **Step 3: Commit specs into new repo** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-extension-spikeinterface -git add superpowers/specs/ -git commit -m "docs: add SI design specs moved from orcapod-python (ITL-473)" -``` - ---- - -### Task 16: Install deps and run tests in new package - -- [ ] **Step 1: Initialize uv project and install deps** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-extension-spikeinterface -uv sync --dev -``` - -- [ ] **Step 2: Run the tests** - -```bash -uv run pytest -v --tb=short -``` - -Expected: All tests pass (spikeinterface is installed as a required dep; tests that previously required `pytest.importorskip("spikeinterface")` will now always run). - -If any test fails due to import path issues, fix the remaining `from orcapod.extension_types.spikeinterface_types import` references in `tests/test_spikeinterface_types.py`. - -- [ ] **Step 3: Verify the public API works end-to-end** - -```bash -uv run python -c " -import orcapod as op -from orcapod_extension_spikeinterface import spikeinterface_extension -op.register_extension(spikeinterface_extension) -from orcapod.contexts import get_default_context -ctx = get_default_context() -print('context_key:', ctx.context_key) -print('register_extension OK') -" -``` - -Expected: -``` -context_key: std:v0.1:default -register_extension OK -``` - ---- - -### Task 17: Push new repo and finalize orcapod-python PR - -- [ ] **Step 1: Push the new repo to GitHub** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-extension-spikeinterface -git push -u origin main -``` - -- [ ] **Step 2: Push any remaining orcapod-python changes** - -```bash -cd /home/kurouto/kurouto-jobs/17b9f34a-40ec-489c-9ccd-6c96c7a9cdf0/orcapod-python -git push origin eywalker/itl-473-split-spikeinterface-extension-into-separate-orcapod -``` - -- [ ] **Step 3: Verify the PR is up-to-date** - -```bash -gh pr view --web -``` - -Confirm all commits appear in the PR diff. diff --git a/superpowers/plans/2026-07-09-node-config-pod-config-split.md b/superpowers/plans/2026-07-09-node-config-pod-config-split.md deleted file mode 100644 index 75aba5f5f..000000000 --- a/superpowers/plans/2026-07-09-node-config-pod-config-split.md +++ /dev/null @@ -1,931 +0,0 @@ -# NodeConfig / PodConfig Split Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Split the old `NodeConfig` (which held `max_concurrency` and `is_result_ephemeral`) into `PodConfig` (pod/executor concern: `max_concurrency`) and a redefined `NodeConfig` (orchestrator concern: `is_result_ephemeral: bool | None`), removing `node_config` from `FunctionPod` and placing it directly on `FunctionJobNode`. - -**Architecture:** Changes flow outward from the type layer: `types.py` first (new `PodConfig`, redefined `NodeConfig`), then `FunctionPod`, then `FunctionJobNode`, then the new `PipelineJob.apply_node_config()` API, then the protocol, and finally all call sites. Each task leaves the test suite passing before the next begins. - -**Tech Stack:** Python 3.12+, `uv run pytest`, `pyarrow`, `asyncio`. - -**Spec:** `superpowers/specs/2026-07-08-node-config-pod-config-split-design.md` - ---- - -## File Map - -| File | Action | What changes | -|---|---|---| -| `src/orcapod/types.py` | Modify | Add `PodConfig`; redefine `NodeConfig` with `is_result_ephemeral: bool \| None` and `merge()`; update `resolve_concurrency()` signature | -| `src/orcapod/core/function_pod.py` | Modify | Replace `node_config`/`_node_config` with `pod_config`/`_pod_config`; update `async_execute`, `to_config`, `from_config` | -| `src/orcapod/core/nodes/function_node.py` | Modify | Add `_node_config` field + `node_config` property; update `_process_data_internal`, `_async_process_data_internal`, `async_execute` | -| `src/orcapod/pipeline/job.py` | Modify | Add `_iter_function_job_nodes()` and `apply_node_config()` | -| `src/orcapod/protocols/node_protocols.py` | Modify | Add `node_config` read/write property to `FunctionNodeProtocol` | -| `tests/test_core/test_node_config.py` | Modify | Replace/add tests for `PodConfig` and new `NodeConfig.merge()` semantics | -| `tests/test_core/function_pod/test_ephemeral_result.py` | Modify | Replace `FunctionPod(pf, node_config=NodeConfig(is_result_ephemeral=True))` with `node.node_config = NodeConfig(...)` | -| `tests/test_channels/test_async_execute.py` | Modify | `NodeConfig(max_concurrency=N)` → `PodConfig(max_concurrency=N)` throughout | -| `tests/test_channels/test_node_async_execute.py` | Modify | Same `max_concurrency` → `PodConfig` migration | -| `tests/test_pipeline/test_orchestrator_executor_matrix.py` | Modify | Same `max_concurrency` → `PodConfig` migration | -| `tests/test_channels/test_copilot_review_issues.py` | Modify | Same `max_concurrency` → `PodConfig` migration | -| `tests/test_channels/test_channels.py` | Modify | Same `max_concurrency` → `PodConfig` migration | -| `tests/test_core/test_regression_fixes.py` | Modify | Same `max_concurrency` → `PodConfig` migration | -| `tests/test_channels/test_pipeline_example.py` | Modify | Same `max_concurrency` → `PodConfig` migration | -| `tests/test_pipeline/test_serialization_helpers.py` | Modify | `node_config` key → `pod_config` in serialised dicts | -| `tests/test_pipeline/test_apply_node_config.py` | Create | New tests for `PipelineJob.apply_node_config()` | - ---- - -## Task 1: Introduce `PodConfig` and redefine `NodeConfig` in `types.py` - -**Files:** -- Modify: `src/orcapod/types.py` (lines 334–372) -- Modify: `tests/test_core/test_node_config.py` - -- [ ] **Step 1: Read the existing node config test file** - -```bash -cat -n tests/test_core/test_node_config.py -``` - -Note which tests currently pass — these will need updating. - -- [ ] **Step 2: Run existing tests to establish baseline** - -```bash -uv run pytest tests/test_core/test_node_config.py -v -``` - -- [ ] **Step 3: Replace the contents of `tests/test_core/test_node_config.py` with updated tests** - -```python -import pytest - -from orcapod.types import NodeConfig, PipelineConfig, PodConfig, resolve_concurrency - - -class TestPodConfig: - def test_defaults(self): - config = PodConfig() - assert config.max_concurrency is None - - def test_max_concurrency(self): - config = PodConfig(max_concurrency=4) - assert config.max_concurrency == 4 - - def test_immutable(self): - config = PodConfig(max_concurrency=4) - with pytest.raises((AttributeError, TypeError)): - config.max_concurrency = 8 # type: ignore[misc] - - -class TestNodeConfig: - def test_defaults(self): - config = NodeConfig() - assert config.is_result_ephemeral is None - - def test_is_result_ephemeral_true(self): - config = NodeConfig(is_result_ephemeral=True) - assert config.is_result_ephemeral is True - - def test_is_result_ephemeral_false(self): - config = NodeConfig(is_result_ephemeral=False) - assert config.is_result_ephemeral is False - - def test_immutable(self): - config = NodeConfig(is_result_ephemeral=True) - with pytest.raises((AttributeError, TypeError)): - config.is_result_ephemeral = False # type: ignore[misc] - - def test_merge_none_in_other_self_wins(self): - """None in other does not override self's value.""" - base = NodeConfig(is_result_ephemeral=True) - result = base.merge(NodeConfig()) - assert result.is_result_ephemeral is True - - def test_merge_non_none_in_other_other_wins(self): - """Non-None in other overrides self.""" - base = NodeConfig(is_result_ephemeral=True) - result = base.merge(NodeConfig(is_result_ephemeral=False)) - assert result.is_result_ephemeral is False - - def test_merge_false_overrides_true(self): - """Explicit False wins over True.""" - base = NodeConfig(is_result_ephemeral=True) - result = base.merge(NodeConfig(is_result_ephemeral=False)) - assert result.is_result_ephemeral is False - - def test_merge_both_none(self): - result = NodeConfig().merge(NodeConfig()) - assert result.is_result_ephemeral is None - - def test_merge_returns_new_instance(self): - base = NodeConfig(is_result_ephemeral=True) - result = base.merge(NodeConfig()) - assert result is not base - - -class TestResolveConcurrency: - def test_pod_config_wins_over_pipeline(self): - pod = PodConfig(max_concurrency=4) - pipeline = PipelineConfig(default_max_concurrency=2) - assert resolve_concurrency(pod, pipeline) == 4 - - def test_falls_back_to_pipeline_when_pod_is_none(self): - pod = PodConfig(max_concurrency=None) - pipeline = PipelineConfig(default_max_concurrency=2) - assert resolve_concurrency(pod, pipeline) == 2 - - def test_both_none_returns_none(self): - pod = PodConfig(max_concurrency=None) - pipeline = PipelineConfig(default_max_concurrency=None) - assert resolve_concurrency(pod, pipeline) is None - - def test_invalid_zero_raises(self): - pod = PodConfig(max_concurrency=0) - with pytest.raises(ValueError, match="max_concurrency must be >= 1"): - resolve_concurrency(pod, PipelineConfig()) - - def test_invalid_negative_raises(self): - pod = PodConfig(max_concurrency=-1) - with pytest.raises(ValueError, match="max_concurrency must be >= 1"): - resolve_concurrency(pod, PipelineConfig()) -``` - -- [ ] **Step 4: Run to confirm tests fail** - -```bash -uv run pytest tests/test_core/test_node_config.py -v -``` - -Expected: FAIL — `PodConfig` not defined, `NodeConfig` still has `max_concurrency`, `merge()` does not exist. - -- [ ] **Step 5: Replace lines 334–372 of `src/orcapod/types.py` with the new types** - -```python -@dataclass(frozen=True, slots=True) -class PodConfig: - """Per-pod executor configuration. - - Attributes: - max_concurrency: Maximum concurrent function invocations for this pod. - ``None`` inherits from ``PipelineConfig.default_max_concurrency``. - ``1`` means sequential (rate-limited APIs, preserves ordering). - """ - - max_concurrency: int | None = None - - -@dataclass(frozen=True, slots=True) -class NodeConfig: - """Per-node pipeline execution configuration. - - Attributes: - is_result_ephemeral: ``None`` inherits the default (``False``). - ``True`` writes new computation results to the pipeline-scoped - ephemeral store instead of the persistent result database. - Persistent cache hits are still served when available. Raises - ``RuntimeError`` at execution time if ``True`` but no ephemeral - store has been injected via ``set_ephemeral_store()``. - """ - - is_result_ephemeral: bool | None = None - - def merge(self, other: "NodeConfig") -> "NodeConfig": - """Return a new ``NodeConfig`` with ``other``'s non-``None`` fields overriding self. - - ``None`` fields in ``other`` are treated as "not set" and leave - self's value unchanged. - - Args: - other: The ``NodeConfig`` whose non-``None`` fields take precedence. - - Returns: - A new immutable ``NodeConfig``. - - Example: - NodeConfig(is_result_ephemeral=True).merge(NodeConfig()) - # → NodeConfig(is_result_ephemeral=True) (other's None leaves self unchanged) - - NodeConfig(is_result_ephemeral=True).merge(NodeConfig(is_result_ephemeral=False)) - # → NodeConfig(is_result_ephemeral=False) (other's explicit False wins) - """ - return NodeConfig( - is_result_ephemeral=( - other.is_result_ephemeral - if other.is_result_ephemeral is not None - else self.is_result_ephemeral - ), - ) - - -def resolve_concurrency( - pod_config: PodConfig, pipeline_config: PipelineConfig -) -> int | None: - """Resolve effective concurrency from pod and pipeline configs. - - Returns: - The concurrency limit to use, or ``None`` for unlimited. - - Raises: - ValueError: If the resolved value is ``<= 0``. - """ - if pod_config.max_concurrency is not None: - result = pod_config.max_concurrency - else: - result = pipeline_config.default_max_concurrency - if result is not None and result <= 0: - raise ValueError(f"max_concurrency must be >= 1, got {result}") - return result -``` - -- [ ] **Step 6: Check whether `PodConfig` needs to be exported from the top-level package** - -```bash -grep -n "NodeConfig" src/orcapod/__init__.py 2>/dev/null || echo "not re-exported" -``` - -If `NodeConfig` is re-exported, add `PodConfig` to the same import line. - -- [ ] **Step 7: Run tests to confirm passing** - -```bash -uv run pytest tests/test_core/test_node_config.py -v -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/types.py tests/test_core/test_node_config.py -git commit -m "feat(types): introduce PodConfig and redefine NodeConfig with merge()" -``` - ---- - -## Task 2: Update `FunctionPod` to use `pod_config` - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` (lines 242–253, 291–340, ~359) -- Modify: `tests/test_channels/test_async_execute.py` (lines 44, 577, 597, 616, 622, 836, 843) -- Modify: `tests/test_pipeline/test_serialization_helpers.py` (line 481) - -- [ ] **Step 1: Run baseline to see current failures caused by Task 1** - -```bash -uv run pytest tests/test_channels/test_async_execute.py tests/test_pipeline/test_serialization_helpers.py -v 2>&1 | grep -E "FAILED|ERROR" | head -20 -``` - -- [ ] **Step 2: Update imports in `src/orcapod/core/function_pod.py`** - -Find the line that imports from `orcapod.types` and ensure it includes `PodConfig`: - -```python -from orcapod.types import NodeConfig, PipelineConfig, PodConfig, resolve_concurrency -``` - -(Check the existing import — it likely already imports `NodeConfig`, `PipelineConfig`, and `resolve_concurrency`. Add `PodConfig`.) - -- [ ] **Step 3: Replace `FunctionPod.__init__()` and add `.pod_config` property** - -Replace lines 242–253 of `src/orcapod/core/function_pod.py`: - -```python -def __init__( - self, - data_function: DataFunctionProtocol, - pod_config: PodConfig | None = None, - **kwargs, -) -> None: - super().__init__(data_function, **kwargs) - self._pod_config = pod_config or PodConfig() - -@property -def pod_config(self) -> PodConfig: - """Per-pod executor configuration.""" - return self._pod_config -``` - -- [ ] **Step 4: Update `async_execute()` at line ~359** - -Change the `resolve_concurrency` call from `self._node_config` to `self._pod_config`: - -```python -# Before (line ~359) -max_concurrency = resolve_concurrency(self._node_config, pipeline_config) - -# After -max_concurrency = resolve_concurrency(self._pod_config, pipeline_config) -``` - -- [ ] **Step 5: Update `to_config()` (lines 291–310)** - -Replace lines 291–310 with: - -```python -def to_config(self) -> dict[str, Any]: - """Serialize this function pod to a JSON-compatible config dict. - - Returns: - A JSON-serializable dict containing the URI, data function config, - and pod config for this function pod. - """ - config: dict[str, Any] = { - "uri": list(self.uri), - "data_function": self.data_function.to_config(), - "pod_config": None, - } - if self._pod_config.max_concurrency is not None: - config["pod_config"] = { - "max_concurrency": self._pod_config.max_concurrency, - } - return config -``` - -- [ ] **Step 6: Update `from_config()` (lines 312–340)** - -Replace lines 312–340 with: - -```python -@classmethod -def from_config( - cls, - config: dict[str, Any], - *, - fallback_to_proxy: bool = False, -) -> "FunctionPod": - """Reconstruct a ``FunctionPod`` from a config dict. - - Args: - config: A dict as produced by ``to_config``. - fallback_to_proxy: If ``True`` and the data function cannot be - resolved, use a ``DataFunctionProxy`` instead of raising. - - Returns: - A new ``FunctionPod`` instance. - """ - from orcapod.pipeline.serialization import resolve_data_function_from_config - - pf_config = config["data_function"] - data_function = resolve_data_function_from_config( - pf_config, fallback_to_proxy=fallback_to_proxy - ) - - pod_config = None - if config.get("pod_config") is not None: - pod_config = PodConfig(**config["pod_config"]) - - return cls(data_function=data_function, pod_config=pod_config) -``` - -- [ ] **Step 7: Update `tests/test_channels/test_async_execute.py` (lines 44, 577, 597, 616, 622, 836, 843)** - -Replace all `NodeConfig(max_concurrency=...)` with `PodConfig(max_concurrency=...)` and update the import: - -```python -# Before -from orcapod.types import NodeConfig -pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=5)) - -# After -from orcapod.types import PodConfig -pod = FunctionPod(pf, pod_config=PodConfig(max_concurrency=5)) -``` - -- [ ] **Step 8: Update `tests/test_pipeline/test_serialization_helpers.py` (line ~481)** - -Update any serialised config dict that uses the `"node_config"` key: - -```python -# Before -{"node_config": {"max_concurrency": 4}, "data_function": ..., "uri": ...} - -# After -{"pod_config": {"max_concurrency": 4}, "data_function": ..., "uri": ...} -``` - -- [ ] **Step 9: Run tests to confirm passing** - -```bash -uv run pytest tests/test_channels/test_async_execute.py tests/test_pipeline/test_serialization_helpers.py -v -``` - -Expected: PASS. - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/core/function_pod.py \ - tests/test_channels/test_async_execute.py \ - tests/test_pipeline/test_serialization_helpers.py -git commit -m "refactor(function_pod): replace node_config with pod_config" -``` - ---- - -## Task 3: Update `FunctionJobNode` to own `node_config` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines ~703–744, ~1203–1207, ~1320–1324, ~2080–2081) -- Create: `tests/test_core/function_pod/test_node_config_on_job_node.py` -- Modify: `tests/test_core/function_pod/test_ephemeral_result.py` (lines 19, 30, 32, 59, 185, 217–218, 247–248, 260, 324–325, 472, 489, 515–516, 528, 541, 563, 584, 687, 691, 693, 722, 724, 759, 761) -- Modify: `tests/test_channels/test_node_async_execute.py` (lines 35, 337, 374, 410) - -- [ ] **Step 1: Write a failing test for the `node_config` property** - -Create `tests/test_core/function_pod/test_node_config_on_job_node.py`: - -```python -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.types import NodeConfig - - -def _make_node() -> FunctionJobNode: - def double(x: int) -> int: - return x * 2 - - pf = PythonDataFunction(double, output_keys="result") - pod = FunctionPod(pf) - table = pa.table({"id": [1], "x": [10]}) - source = ArrowTableSource(table, tag_columns=["id"]) - db = InMemoryArrowDatabase() - return FunctionJobNode(pod, source, pipeline_database=db) - - -class TestFunctionJobNodeNodeConfig: - def test_default_node_config(self): - """FunctionJobNode initialises with NodeConfig() as default.""" - node = _make_node() - assert isinstance(node.node_config, NodeConfig) - assert node.node_config.is_result_ephemeral is None - - def test_node_config_setter(self): - """node_config property setter replaces the config.""" - node = _make_node() - new_config = NodeConfig(is_result_ephemeral=True) - node.node_config = new_config - assert node.node_config is new_config - - def test_set_then_replace(self): - """Setting node_config twice replaces it cleanly.""" - node = _make_node() - node.node_config = NodeConfig(is_result_ephemeral=True) - node.node_config = NodeConfig(is_result_ephemeral=False) - assert node.node_config.is_result_ephemeral is False - - def test_ephemeral_false_resolves_correctly(self): - """is_result_ephemeral=None resolves to False at execution.""" - node = _make_node() - # None should resolve to False via `or False` - assert (node.node_config.is_result_ephemeral or False) is False - - def test_ephemeral_true_resolves_correctly(self): - """is_result_ephemeral=True resolves to True at execution.""" - node = _make_node() - node.node_config = NodeConfig(is_result_ephemeral=True) - assert (node.node_config.is_result_ephemeral or False) is True -``` - -- [ ] **Step 2: Run to confirm failure** - -```bash -uv run pytest tests/test_core/function_pod/test_node_config_on_job_node.py -v -``` - -Expected: FAIL — `FunctionJobNode` has no `node_config` attribute. - -- [ ] **Step 3: Update imports in `src/orcapod/core/nodes/function_node.py`** - -Find the import from `orcapod.types` and ensure it includes `NodeConfig` and `PodConfig`: - -```python -from orcapod.types import NodeConfig, PipelineConfig, PodConfig, resolve_concurrency -``` - -- [ ] **Step 4: Add `_node_config` field to `FunctionJobNode.__init__()`** - -In `FunctionJobNode.__init__()` (around line 740, before `self.attach_databases(...)`), add: - -```python -# Node-level orchestrator config — applied post-construction via the -# .node_config setter or PipelineJob.apply_node_config(). -self._node_config: NodeConfig = NodeConfig() -``` - -- [ ] **Step 5: Add the `node_config` property to `FunctionJobNode`** - -Immediately after `__init__`, add: - -```python -@property -def node_config(self) -> NodeConfig: - """Per-node pipeline execution configuration.""" - return self._node_config - -@node_config.setter -def node_config(self, value: NodeConfig) -> None: - self._node_config = value -``` - -- [ ] **Step 6: Update `_process_data_internal()` (lines ~1203–1207)** - -Replace: - -```python -# Before -node_config = ( - self._function_pod.node_config if self._function_pod is not None else None -) -ephemeral_result = ( - node_config.is_result_ephemeral if node_config is not None else False -) - -# After -ephemeral_result = self._node_config.is_result_ephemeral or False -``` - -- [ ] **Step 7: Update `_async_process_data_internal()` (lines ~1320–1324)** - -Identical replacement as Step 6 — same two lines, same fix: - -```python -# Before -node_config = ( - self._function_pod.node_config if self._function_pod is not None else None -) -ephemeral_result = ( - node_config.is_result_ephemeral if node_config is not None else False -) - -# After -ephemeral_result = self._node_config.is_result_ephemeral or False -``` - -- [ ] **Step 8: Update `async_execute()` (lines ~2080–2081)** - -Replace: - -```python -# Before -node_config = getattr(self._function_pod, "node_config", NodeConfig()) -max_concurrency = resolve_concurrency(node_config, PipelineConfig()) - -# After -pod_config = getattr(self._function_pod, "pod_config", PodConfig()) -max_concurrency = resolve_concurrency(pod_config, PipelineConfig()) -``` - -- [ ] **Step 9: Run the new property tests** - -```bash -uv run pytest tests/test_core/function_pod/test_node_config_on_job_node.py -v -``` - -Expected: PASS. - -- [ ] **Step 10: Update `tests/test_core/function_pod/test_ephemeral_result.py`** - -Every place that currently passes `is_result_ephemeral=True` to `FunctionPod` must change to set it on the node after construction. Pattern: - -```python -# Before -pod = FunctionPod(pf, node_config=NodeConfig(is_result_ephemeral=True)) -node = FunctionJobNode(pod, stream, pipeline_database=db, ephemeral_database=edb) - -# After -pod = FunctionPod(pf) -node = FunctionJobNode(pod, stream, pipeline_database=db, ephemeral_database=edb) -node.node_config = NodeConfig(is_result_ephemeral=True) -``` - -Apply this pattern to ALL usages at lines: 19, 30, 32, 59, 185, 217–218, 247–248, 260, 324–325, 472, 489, 515–516, 528, 541, 563, 584, 687, 691, 693, 722, 724, 759, 761. - -Also update the import in this file — remove `NodeConfig` from the `FunctionPod` import context if it was only used as a constructor argument, and keep it for direct `NodeConfig(...)` usage. - -- [ ] **Step 11: Update `tests/test_channels/test_node_async_execute.py` (lines 35, 337, 374, 410)** - -Same pattern as Step 10 — remove `node_config=NodeConfig(is_result_ephemeral=True)` from `FunctionPod(...)` and set `node.node_config = NodeConfig(is_result_ephemeral=True)` on the job node after construction. - -- [ ] **Step 12: Run all affected tests** - -```bash -uv run pytest tests/test_core/function_pod/ tests/test_channels/test_node_async_execute.py -v -``` - -Expected: PASS. - -- [ ] **Step 13: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py \ - tests/test_core/function_pod/ \ - tests/test_channels/test_node_async_execute.py -git commit -m "refactor(function_node): move node_config ownership to FunctionJobNode" -``` - ---- - -## Task 4: Add `PipelineJob.apply_node_config()` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` -- Create: `tests/test_pipeline/test_apply_node_config.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_pipeline/test_apply_node_config.py`: - -```python -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.pipeline import PipelineJob -from orcapod.types import NodeConfig - - -def _make_job() -> PipelineJob: - """Two-node pipeline: double → triple.""" - - def double(x: int) -> int: - return x * 2 - - def triple(x: int) -> int: - return x * 3 - - pf1 = PythonDataFunction(double, output_keys="doubled") - pf2 = PythonDataFunction(triple, output_keys="tripled") - pod1 = FunctionPod(pf1) - pod2 = FunctionPod(pf2) - db = InMemoryArrowDatabase() - table = pa.table({"id": [1, 2], "x": [10, 20]}) - source = ArrowTableSource(table, tag_columns=["id"]) - job = PipelineJob(name="test_job", store=db) - with job: - out1 = pod1(source) - pod2(out1) - return job - - -class TestApplyNodeConfig: - def test_apply_sets_config_on_all_nodes(self): - """apply_node_config sets config on every FunctionJobNode.""" - job = _make_job() - config = NodeConfig(is_result_ephemeral=True) - job.apply_node_config(config) - nodes = list(job._iter_function_job_nodes()) - assert len(nodes) == 2 - for node in nodes: - assert node.node_config.is_result_ephemeral is True - - def test_apply_replace_existing_false_wholesale(self): - """replace_existing=False replaces each node's config wholesale.""" - job = _make_job() - nodes = list(job._iter_function_job_nodes()) - # Pre-set one node to ephemeral=True - nodes[0].node_config = NodeConfig(is_result_ephemeral=True) - # Apply a new config wholesale — replaces whatever each node had - job.apply_node_config(NodeConfig(is_result_ephemeral=False), replace_existing=False) - for node in job._iter_function_job_nodes(): - assert node.node_config.is_result_ephemeral is False - - def test_apply_replace_existing_true_merges_none_no_op(self): - """replace_existing=True: None in new config leaves existing values untouched.""" - job = _make_job() - nodes = list(job._iter_function_job_nodes()) - nodes[0].node_config = NodeConfig(is_result_ephemeral=True) - # Merge with a config that has None — should not override True - job.apply_node_config(NodeConfig(), replace_existing=True) - assert nodes[0].node_config.is_result_ephemeral is True - - def test_apply_replace_existing_true_overrides_with_explicit_value(self): - """replace_existing=True: explicit False in new config overrides existing True.""" - job = _make_job() - nodes = list(job._iter_function_job_nodes()) - nodes[0].node_config = NodeConfig(is_result_ephemeral=True) - job.apply_node_config(NodeConfig(is_result_ephemeral=False), replace_existing=True) - assert nodes[0].node_config.is_result_ephemeral is False - - def test_iter_function_job_nodes_yields_only_function_nodes(self): - """_iter_function_job_nodes does not yield source or operator nodes.""" - from orcapod.core.nodes.function_node import FunctionJobNode - job = _make_job() - nodes = list(job._iter_function_job_nodes()) - assert all(isinstance(n, FunctionJobNode) for n in nodes) -``` - -- [ ] **Step 2: Run to confirm failure** - -```bash -uv run pytest tests/test_pipeline/test_apply_node_config.py -v -``` - -Expected: FAIL — `apply_node_config` and `_iter_function_job_nodes` not defined on `PipelineJob`. - -- [ ] **Step 3: Add imports to `src/orcapod/pipeline/job.py`** - -At the top of `job.py`, add: - -```python -from collections.abc import Iterator - -from orcapod.types import NodeConfig -``` - -And inside the method (lazy import to avoid circularity): - -```python -from orcapod.core.nodes.function_node import FunctionJobNode -``` - -- [ ] **Step 4: Add `_iter_function_job_nodes()` to `PipelineJob`** - -```python -def _iter_function_job_nodes(self) -> "Iterator[FunctionJobNode]": - """Iterate over all ``FunctionJobNode`` instances in the compiled pipeline. - - Returns: - An iterator over every ``FunctionJobNode`` in the persistent node map. - """ - from orcapod.core.nodes.function_node import FunctionJobNode - - return ( - node - for node in self._persistent_node_map.values() - if isinstance(node, FunctionJobNode) - ) -``` - -- [ ] **Step 5: Add `apply_node_config()` to `PipelineJob`** - -```python -def apply_node_config( - self, - node_config: NodeConfig, - replace_existing: bool = False, -) -> None: - """Apply a ``NodeConfig`` to all ``FunctionJobNode``s in this pipeline. - - Args: - node_config: The config to apply. - replace_existing: If ``False``, directly sets ``node_config`` on - every node, replacing whatever config each node currently holds. - If ``True``, merges ``node_config`` into each node's existing - config — non-``None`` fields in ``node_config`` override the - node's current values, while ``None`` fields leave existing - values unchanged. - """ - for node in self._iter_function_job_nodes(): - if replace_existing: - node.node_config = node.node_config.merge(node_config) - else: - node.node_config = node_config -``` - -- [ ] **Step 6: Run tests to confirm passing** - -```bash -uv run pytest tests/test_pipeline/test_apply_node_config.py -v -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/pipeline/job.py tests/test_pipeline/test_apply_node_config.py -git commit -m "feat(pipeline): add PipelineJob.apply_node_config() and _iter_function_job_nodes()" -``` - ---- - -## Task 5: Update `FunctionNodeProtocol` - -**Files:** -- Modify: `src/orcapod/protocols/node_protocols.py` - -- [ ] **Step 1: Add `node_config` read/write property to `FunctionNodeProtocol`** - -Open `src/orcapod/protocols/node_protocols.py`. In the `FunctionNodeProtocol` class, add the following after the existing `pipeline_path` property and before `execute`: - -```python -@property -def node_config(self) -> "NodeConfig": ... - -@node_config.setter -def node_config(self, value: "NodeConfig") -> None: ... -``` - -In the `TYPE_CHECKING` block (or regular imports) at the top of the file, ensure `NodeConfig` is imported: - -```python -if TYPE_CHECKING: - from orcapod.types import NodeConfig -``` - -If `NodeConfig` is already a runtime import in this file, add it there instead. - -- [ ] **Step 2: Run protocol-related tests** - -```bash -uv run pytest tests/ -v -k "protocol" 2>&1 | tail -20 -``` - -Expected: PASS (structural change, no runtime behaviour changed). - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/protocols/node_protocols.py -git commit -m "refactor(protocols): add node_config property to FunctionNodeProtocol" -``` - ---- - -## Task 6: Fix remaining test construction sites - -**Files:** -- Modify: `tests/test_pipeline/test_orchestrator_executor_matrix.py` (lines 47, 78) -- Modify: `tests/test_channels/test_copilot_review_issues.py` (lines 31, 86, 244, 248, 252, 256, 260, 266, 278) -- Modify: `tests/test_channels/test_channels.py` (lines 572–575, 579–595) -- Modify: `tests/test_core/test_regression_fixes.py` (lines 40, 330) -- Modify: `tests/test_channels/test_pipeline_example.py` (lines 36, 233) - -- [ ] **Step 1: Run the full suite to see all remaining failures** - -```bash -uv run pytest tests/ -v 2>&1 | grep -E "FAILED|ERROR" -``` - -- [ ] **Step 2: Update `tests/test_pipeline/test_orchestrator_executor_matrix.py` (lines 47, 78)** - -```python -# Before -from orcapod.types import NodeConfig -pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=max_concurrency)) - -# After -from orcapod.types import PodConfig -pod = FunctionPod(pf, pod_config=PodConfig(max_concurrency=max_concurrency)) -``` - -- [ ] **Step 3: Update `tests/test_channels/test_copilot_review_issues.py` (lines 31, 86, 244, 248, 252, 256, 260, 266, 278)** - -```python -# Before -from orcapod.types import NodeConfig -pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=5)) - -# After -from orcapod.types import PodConfig -pod = FunctionPod(pf, pod_config=PodConfig(max_concurrency=5)) -``` - -- [ ] **Step 4: Update `tests/test_channels/test_channels.py` (lines 572–575, 579–595)** - -```python -# Before -from orcapod.types import NodeConfig -FunctionPod(pf, node_config=NodeConfig(max_concurrency=N)) - -# After -from orcapod.types import PodConfig -FunctionPod(pf, pod_config=PodConfig(max_concurrency=N)) -``` - -- [ ] **Step 5: Update `tests/test_core/test_regression_fixes.py` (lines 40, 330)** - -Same pattern as Step 4. - -- [ ] **Step 6: Update `tests/test_channels/test_pipeline_example.py` (lines 36, 233)** - -Same pattern as Step 4. - -- [ ] **Step 7: Run the full test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: all tests PASS. - -- [ ] **Step 8: Commit** - -```bash -git add tests/test_pipeline/test_orchestrator_executor_matrix.py \ - tests/test_channels/test_copilot_review_issues.py \ - tests/test_channels/test_channels.py \ - tests/test_core/test_regression_fixes.py \ - tests/test_channels/test_pipeline_example.py -git commit -m "refactor(tests): migrate all NodeConfig(max_concurrency) usages to PodConfig" -``` diff --git a/superpowers/plans/2026-07-10-itl-516-unify-async-execute.md b/superpowers/plans/2026-07-10-itl-516-unify-async-execute.md deleted file mode 100644 index f1e3cc046..000000000 --- a/superpowers/plans/2026-07-10-itl-516-unify-async-execute.md +++ /dev/null @@ -1,1357 +0,0 @@ -# ITL-516: Unify FunctionPod/FunctionJobNode async_execute() Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Move `async_execute()` to `_FunctionPodBase`, redesign `FunctionJobNode.async_execute()` as a 3-stage concurrent pipeline (route → compute → record), eliminating duplicate dispatch loops. - -**Architecture:** `_FunctionPodBase.async_execute()` becomes the single authoritative async dispatch loop (with observer support), used by all pod types via polymorphism. `FunctionJobNode.async_execute()` delegates computation to `execution_pod.async_execute()` via bounded `compute_channel`/`result_channel`, retaining only routing (cache hits vs misses), correlation-key stamping, `add_pipeline_record` calls, and key stripping. Three coroutines run concurrently in a `TaskGroup`. - -**Tech Stack:** Python asyncio (`asyncio.TaskGroup`, `asyncio.Semaphore`), `orcapod.channels.Channel`, `pytest-asyncio`. - ---- - -## File Structure - -| File | Action | Responsibility | -|------|--------|----------------| -| `src/orcapod/core/function_pod.py` | Modify | Add `pod_config` to `_FunctionPodBase` and `WrappedFunctionPod`; move `async_execute()` from `FunctionPod` to `_FunctionPodBase` with observer hooks | -| `src/orcapod/core/nodes/function_node.py` | Modify | Add `_TAG_NODE_INPUT_REF` constant; replace `FunctionJobNode.async_execute()` with 3-stage pipeline; delete `_async_execute_one_data`; remove dead imports | -| `tests/test_core/test_regression_fixes.py` | Modify | Remove `TestAsyncExecuteBackpressure` (orphan test for orphan method) | -| `tests/test_core/nodes/test_function_job_node_async_execute.py` | Create | All async_execute coverage: no-DB, DB (persistent + ephemeral), cache hits/misses, correlation key absence, concurrency, observer hooks | - ---- - -## Task 1: Move `async_execute()` to `_FunctionPodBase` with observer support - -`FunctionPod.async_execute()` currently lives on `FunctionPod` (lines 344–383 of `function_pod.py`). -`CachedFunctionPod` inherits from `WrappedFunctionPod → _FunctionPodBase`, not from `FunctionPod`, so -it currently has no `async_execute()`. Moving the method to `_FunctionPodBase` gives all pod types -the dispatch loop via polymorphism. - -Two supporting changes are needed first: a `pod_config` property on `_FunctionPodBase` (default -`PodConfig()`) and a delegating override on `WrappedFunctionPod`. Without these, `CachedFunctionPod` -would silently ignore concurrency limits set on the inner `FunctionPod`. - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` - -- [ ] **Step 1: Write a failing test for observer hooks on `_FunctionPodBase.async_execute()`** - -Create a temporary test in an existing test file or a scratch location. We'll move this to the -dedicated file in Task 4. For now, paste it into the bottom of -`tests/test_core/test_regression_fixes.py` (we'll relocate in Task 4). - -```python -# Add these imports at the top of test_regression_fixes.py if not present: -# from orcapod.types import PodConfig - -class TestFunctionPodBaseAsyncExecuteObserver: - """Observer hooks fire correctly in _FunctionPodBase.async_execute().""" - - @pytest.mark.asyncio - async def test_observer_on_data_start_and_end_fire_per_item(self): - """on_data_start fires before and on_data_end fires after each item.""" - starts = [] - ends = [] - - class _Spy: - def contextualize(self, *path): - return self - def on_node_start(self, *a, **kw): pass - def on_node_end(self, *a, **kw): pass - def on_data_start(self, label, tag, data): - starts.append((label, tag, data)) - def on_data_end(self, label, tag, inp, out, *, cached): - ends.append((label, tag, inp, out, cached)) - def on_data_crash(self, label, tag, data, exc): - pass - def create_data_logger(self, tag, data): - return None - - def double(x: int) -> int: - return x * 2 - - pf = PythonDataFunction(double, output_keys="result") - pod = FunctionPod(pf) - - stream = make_stream(3) - input_ch = Channel(buffer_size=16) - output_ch = Channel(buffer_size=16) - await feed_stream_to_channel(stream, input_ch) - - spy = _Spy() - await pod.async_execute([input_ch.reader], output_ch.writer, observer=spy) - - results = await output_ch.reader.collect() - assert len(results) == 3 - assert len(starts) == 3 - assert len(ends) == 3 - # All items were computed, so cached=False - assert all(cached is False for _, _, _, _, cached in ends) - - @pytest.mark.asyncio - async def test_observer_on_data_crash_fires_on_failure(self): - """on_data_crash fires (not on_data_end) when async_process_data raises.""" - crashes = [] - ends = [] - - class _Spy: - def contextualize(self, *path): return self - def on_node_start(self, *a, **kw): pass - def on_node_end(self, *a, **kw): pass - def on_data_start(self, *a): pass - def on_data_end(self, label, tag, inp, out, *, cached): - ends.append((label, tag, inp, out, cached)) - def on_data_crash(self, label, tag, data, exc): - crashes.append(exc) - def create_data_logger(self, tag, data): - return None - - def boom(x: int) -> int: - raise ValueError("explode") - - pf = PythonDataFunction(boom, output_keys="result") - pod = FunctionPod(pf) - - stream = make_stream(2) - input_ch = Channel(buffer_size=16) - output_ch = Channel(buffer_size=16) - await feed_stream_to_channel(stream, input_ch) - - spy = _Spy() - await pod.async_execute([input_ch.reader], output_ch.writer, observer=spy) - - results = await output_ch.reader.collect() - assert len(results) == 0 # all failed → nothing forwarded - assert len(crashes) == 2 # crash fires per failed item - assert len(ends) == 0 # on_data_end never fires on crash -``` - -- [ ] **Step 2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_core/test_regression_fixes.py::TestFunctionPodBaseAsyncExecuteObserver -v -``` - -Expected: `FAILED` — `pod.async_execute()` currently accepts no `observer` parameter. - -- [ ] **Step 3: Add `pod_config` default to `_FunctionPodBase` and delegating override to `WrappedFunctionPod`** - -In `src/orcapod/core/function_pod.py`, locate `_FunctionPodBase` (line 60) and add the property -**after** the `executor` setter (around line 103): - -```python -# --- add to _FunctionPodBase, after the executor setter --- -@property -def pod_config(self) -> PodConfig: - """Per-pod executor configuration. Defaults to no concurrency limits.""" - return PodConfig() -``` - -Then locate `WrappedFunctionPod` (line 750) and add a delegating override **after** `computed_label`: - -```python -# --- add to WrappedFunctionPod, after computed_label --- -@property -def pod_config(self) -> PodConfig: - """Delegate to the inner pod's config so CachedFunctionPod respects limits.""" - return getattr(self._function_pod, "pod_config", PodConfig()) -``` - -- [ ] **Step 4: Add `ExecutionObserverProtocol` import to `function_pod.py`** - -`function_pod.py` currently doesn't import `ExecutionObserverProtocol`. Add it to the `TYPE_CHECKING` -block (it's only used in a type hint): - -```python -# In the TYPE_CHECKING block at the top of function_pod.py, add: -if TYPE_CHECKING: - ... - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol -``` - -- [ ] **Step 5: Move `async_execute()` from `FunctionPod` to `_FunctionPodBase` with observer support** - -Remove the existing `async_execute()` method from `FunctionPod` (lines 340–383, starting from the -`# Async channel execution (streaming mode)` section comment). - -Then add the following method to `_FunctionPodBase` **just before the abstract `process()` method** -(around line 196 in the original file — adjust for line shifts from Step 3): - -```python -async def async_execute( - self, - inputs: "Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]]", - output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", - pipeline_config: PipelineConfig | None = None, - *, - observer: "ExecutionObserverProtocol | None" = None, -) -> None: - """Streaming async execution with per-data concurrency control. - - Each input (tag, data) is dispatched as an independent async task. - A semaphore limits how many tasks are in-flight concurrently. - Observer hooks fire per item: ``on_data_start`` before processing, - ``on_data_end(cached=False)`` on success, ``on_data_crash`` on error. - - Args: - inputs: Single-element sequence containing the input channel. - output: Writable channel for output (tag, data) pairs. - pipeline_config: Optional pipeline-level concurrency config. - observer: Optional observer for per-item lifecycle hooks. - """ - from orcapod.pipeline.observer import NoOpObserver - - try: - pipeline_config = pipeline_config or PipelineConfig() - max_concurrency = resolve_concurrency(self.pod_config, pipeline_config) - obs = observer if observer is not None else NoOpObserver() - pod_label = self.label - - sem = ( - asyncio.Semaphore(max_concurrency) - if max_concurrency is not None - else None - ) - - async def process_one(tag: TagProtocol, data: DataProtocol) -> None: - obs.on_data_start(pod_label, tag, data) - try: - out_tag, result_data = await self.async_process_data(tag, data) - except Exception as exc: - logger.debug( - "Data processing failed, skipping: %s", exc, exc_info=True - ) - obs.on_data_crash(pod_label, tag, data, exc) - else: - obs.on_data_end(pod_label, tag, data, result_data, cached=False) - if result_data is not None: - await output.send((out_tag, result_data)) - finally: - if sem is not None: - sem.release() - - async with asyncio.TaskGroup() as tg: - async for tag, data in inputs[0]: - if sem is not None: - await sem.acquire() - tg.create_task(process_one(tag, data)) - finally: - await output.close() -``` - -- [ ] **Step 6: Run ALL tests to verify nothing broke** - -```bash -uv run pytest tests/ -v --timeout=30 -``` - -Expected: all previously-passing tests still pass, the new observer tests pass. - -Key tests to verify: -- `tests/test_core/test_regression_fixes.py::TestAsyncExecuteChannelCloseOnError` — still passes (FunctionPod inherits) -- `tests/test_core/test_regression_fixes.py::TestAsyncExecuteBackpressure` — still passes (FunctionPod still works) -- `tests/test_core/test_regression_fixes.py::TestFunctionPodBaseAsyncExecuteObserver` — now passes - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/function_pod.py tests/test_core/test_regression_fixes.py -git commit -m "feat(function_pod): move async_execute to _FunctionPodBase with observer support - -Promotes async_execute() from FunctionPod to _FunctionPodBase so all pod -types (FunctionPod, CachedFunctionPod) inherit the dispatch loop. Adds -observer hooks (on_data_start, on_data_end, on_data_crash) per item. -Adds pod_config property to _FunctionPodBase (default PodConfig()) and -WrappedFunctionPod (delegates to inner pod) so concurrency limits are -honoured through CachedFunctionPod. - -Closes ITL-516 partial." -``` - ---- - -## Task 2: `FunctionJobNode.async_execute()` — no-DB delegation path - -Currently the no-DB path in `FunctionJobNode.async_execute()` (lines 2147–2166) reimplements -the same semaphore-guarded TaskGroup loop that's already in `_FunctionPodBase.async_execute()`. -Replace it with a single delegation call. This mirrors the OperatorNode pattern. - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` - -- [ ] **Step 1: Write tests for the no-DB async delegation** - -Create `tests/test_core/nodes/test_function_job_node_async_execute.py` with just the no-DB tests: - -```python -"""Tests for FunctionJobNode.async_execute() — unified async execution path.""" - -from __future__ import annotations - -import asyncio -from typing import Any - -import pyarrow as pa -import pytest - -from orcapod.channels import Channel -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.types import PodConfig - - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- - - -def _make_source(n: int = 3) -> ArrowTableSource: - """ArrowTableSource with tag=key (large_string), data=value (int64).""" - table = pa.table( - { - "key": pa.array([f"k{i}" for i in range(n)], type=pa.large_string()), - "value": pa.array(list(range(n)), type=pa.int64()), - } - ) - return ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - - -def _make_no_db_node(n: int = 3, max_concurrency: int | None = None) -> FunctionJobNode: - """FunctionJobNode with no DB attached.""" - src = _make_source(n) - pod = FunctionPod( - PythonDataFunction(lambda value: value * 2, output_keys="result"), - pod_config=PodConfig(max_concurrency=max_concurrency), - ) - return FunctionJobNode(pod, src) - - -async def _run_node(node: FunctionJobNode) -> list[tuple]: - """Feed the node's own input stream through async_execute and collect results.""" - input_ch: Channel = Channel(buffer_size=32) - output_ch: Channel = Channel(buffer_size=32) - - async def feed(): - for tag, data in node._input_stream.iter_data(): - await input_ch.writer.send((tag, data)) - await input_ch.writer.close() - - await asyncio.gather( - feed(), - node.async_execute(input_ch.reader, output_ch.writer), - ) - return await output_ch.reader.collect() - - -# --------------------------------------------------------------------------- -# No-DB path -# --------------------------------------------------------------------------- - - -class TestFunctionJobNodeAsyncExecuteNoDB: - @pytest.mark.asyncio - async def test_all_items_processed_and_forwarded(self): - """All input items reach the output channel.""" - node = _make_no_db_node(n=3) - results = await _run_node(node) - assert len(results) == 3 - values = sorted(data.as_dict()["result"] for _, data in results) - assert values == [0, 2, 4] - - @pytest.mark.asyncio - async def test_output_channel_closed_after_execution(self): - """Output channel is closed even if execution completes without error.""" - node = _make_no_db_node(n=2) - input_ch: Channel = Channel(buffer_size=8) - output_ch: Channel = Channel(buffer_size=8) - - async def feed(): - for tag, data in node._input_stream.iter_data(): - await input_ch.writer.send((tag, data)) - await input_ch.writer.close() - - await asyncio.gather(feed(), node.async_execute(input_ch.reader, output_ch.writer)) - # collect() should return immediately (channel closed), not hang. - results = await output_ch.reader.collect() - assert isinstance(results, list) - - @pytest.mark.asyncio - async def test_max_concurrency_limits_tasks(self): - """With max_concurrency=1, at most one task runs at a time.""" - concurrent_count = 0 - max_observed = 0 - - async def slow_double(value: int) -> int: - nonlocal concurrent_count, max_observed - concurrent_count += 1 - max_observed = max(max_observed, concurrent_count) - await asyncio.sleep(0.01) - concurrent_count -= 1 - return value * 2 - - pf = PythonDataFunction(slow_double, output_keys="result") - # Patch async_call to be the async function - import functools - original_async_call = pf.async_call - - async def patched_async_call(data, **kwargs): - nonlocal concurrent_count, max_observed - concurrent_count += 1 - max_observed = max(max_observed, concurrent_count) - await asyncio.sleep(0.01) - concurrent_count -= 1 - return await original_async_call(data, **kwargs) - - pf.async_call = patched_async_call # type: ignore[method-assign] - - src = _make_source(5) - pod = FunctionPod(pf, pod_config=PodConfig(max_concurrency=1)) - node = FunctionJobNode(pod, src) - results = await _run_node(node) - assert len(results) == 5 - assert max_observed <= 1 -``` - -- [ ] **Step 2: Run the tests to confirm they pass (they should with existing code)** - -```bash -uv run pytest tests/test_core/nodes/test_function_job_node_async_execute.py::TestFunctionJobNodeAsyncExecuteNoDB -v -``` - -Expected: all 3 tests PASS (existing code already handles no-DB). If any fail, investigate before -proceeding. - -- [ ] **Step 3: Replace the no-DB path in `FunctionJobNode.async_execute()` with delegation** - -In `src/orcapod/core/nodes/function_node.py`, find the else branch at lines 2146–2166: - -```python -else: - # Simple async execution without DB - async with asyncio.TaskGroup() as tg: - async for tag, data in input_channel: - async def _guarded_simple( - t: TagProtocol = tag, p: DataProtocol = data - ) -> None: - try: - await self._async_execute_one_data( - t, p, output, - observer=ctx_obs, - node_label=node_label, - node_hash=node_hash, - ) - finally: - if sem is not None: - sem.release() - - if sem is not None: - await sem.acquire() - tg.create_task(_guarded_simple()) -``` - -Replace the entire else block with: - -```python -else: - # No-DB path: delegate dispatch entirely to the pod. - await self._function_pod.async_execute( - [input_channel], output, observer=ctx_obs - ) - return # output is already closed by the pod's finally; skip outer finally -``` - -Wait — there is a problem here. The outer `finally: await output.close()` will try to close the -output channel a second time (it was already closed by `_function_pod.async_execute()`). Looking -at `_ChannelWriter.close()`: - -```python -async def close(self) -> None: - if not self._channel._closed.is_set(): - self._channel._closed.set() - await self._channel._queue.put(_CLOSED) -``` - -It's idempotent (guarded by `_closed.is_set()`). Double-close is safe. So `return` is not needed; -just replace the else block: - -```python -else: - # No-DB path: delegate dispatch entirely to the pod. - # The pod's async_execute() closes `output` in its own finally. - await self._function_pod.async_execute( - [input_channel], output, observer=ctx_obs - ) -``` - -The outer `finally: await output.close()` will be a no-op since the channel is already closed. - -Also remove the semaphore setup that now only applies to the DB path (lines 2089–2097): - -```python -# REMOVE these lines from async_execute(): -pod_config = getattr(self._function_pod, "pod_config", PodConfig()) -max_concurrency = resolve_concurrency(pod_config, PipelineConfig()) - -sem = ( - asyncio.Semaphore(max_concurrency) - if max_concurrency is not None - else None -) -``` - -- [ ] **Step 4: Delete `_async_execute_one_data`** - -Find and delete the entire `_async_execute_one_data` method (lines 2172–2201): - -```python -async def _async_execute_one_data( - self, - tag: TagProtocol, - data: DataProtocol, - output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", - *, - observer: ExecutionObserverProtocol, - node_label: str, - node_hash: str, -) -> None: - ... -``` - -Delete this method entirely. - -- [ ] **Step 5: Remove dead imports from `function_node.py`** - -After removing the semaphore setup from `async_execute()`, `PipelineConfig`, `PodConfig`, and -`resolve_concurrency` are no longer referenced in `function_node.py`. Remove them from the import: - -Before: -```python -from orcapod.types import ( - ColumnConfig, - ContentHash, - NodeConfig, - PipelineConfig, - PodConfig, - Schema, - resolve_concurrency, -) -``` - -After: -```python -from orcapod.types import ( - ColumnConfig, - ContentHash, - NodeConfig, - Schema, -) -``` - -Verify by running a grep to confirm neither `PipelineConfig`, `PodConfig`, nor `resolve_concurrency` -appear elsewhere in the file: - -```bash -grep -n "PipelineConfig\|PodConfig\|resolve_concurrency" \ - src/orcapod/core/nodes/function_node.py -``` - -Expected: no output. - -- [ ] **Step 6: Run all tests** - -```bash -uv run pytest tests/ -v --timeout=30 -``` - -Expected: all tests still pass, including `TestFunctionJobNodeAsyncExecuteNoDB`. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py \ - tests/test_core/nodes/test_function_job_node_async_execute.py -git commit -m "refactor(function_node): delegate no-DB async_execute to pod - -FunctionJobNode.async_execute() no longer reimplements the semaphore -+TaskGroup dispatch loop for the no-DB path. It delegates directly to -self._function_pod.async_execute(), matching the OperatorNode pattern. -Deletes _async_execute_one_data (dead after delegation). Removes the -now-dead PipelineConfig, PodConfig, and resolve_concurrency imports." -``` - ---- - -## Task 3: `FunctionJobNode.async_execute()` — DB 3-stage concurrent pipeline - -Redesign the DB path into three concurrent coroutines: - -1. **`route_inputs`** — routes cache hits directly to output, stamps cache misses with a - correlation key and forwards to `compute_channel`. -2. **`execution_pod.async_execute()`** — reads `compute_channel`, writes `result_channel` - (this is now `_FunctionPodBase.async_execute()`). -3. **`record_and_forward`** — reads `result_channel`, calls `add_pipeline_record`, strips the - correlation key from the output tag, writes to `output`. - -Covers both the persistent (`_cached_function_pod`, `is_ephemeral=False`) and ephemeral -(`_ephemeral_cached_pod`, `is_ephemeral=True`) sub-paths — only the pod and the flag differ. - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` -- Modify: `tests/test_core/nodes/test_function_job_node_async_execute.py` - -- [ ] **Step 1: Write failing tests for the DB path** - -Add the following test classes to `tests/test_core/nodes/test_function_job_node_async_execute.py`. - -First, add the `_TAG_NODE_INPUT_REF` constant import at the top of the test file so we can assert -the correlation key is absent: - -```python -# Add to imports at the top: -from orcapod.core.nodes.function_node import _TAG_NODE_INPUT_REF # private constant -from orcapod.system_constants import constants as _constants -``` - -Then add these test classes at the bottom of the file: - -```python -# --------------------------------------------------------------------------- -# Helpers — observer spy -# --------------------------------------------------------------------------- - - -class _SpyObserver: - """Captures all observer hook invocations.""" - - def __init__(self): - self.node_starts: list[str] = [] - self.node_ends: list[str] = [] - self.data_starts: list[tuple] = [] - self.data_ends: list[tuple] = [] # (label, tag, input_data, output_data, cached) - self.data_crashes: list[tuple] = [] - - def contextualize(self, *path): - return self - - def on_node_start(self, node_label, node_hash, *, tag_schema=None): - self.node_starts.append(node_label) - - def on_node_end(self, node_label, node_hash): - self.node_ends.append(node_label) - - def on_data_start(self, node_label, tag, data): - self.data_starts.append((node_label, tag, data)) - - def on_data_end(self, node_label, tag, input_data, output_data, *, cached): - self.data_ends.append((node_label, tag, input_data, output_data, cached)) - - def on_data_crash(self, node_label, tag, data, error): - self.data_crashes.append((node_label, tag, data, error)) - - def create_data_logger(self, tag, data): - return None - - -# --------------------------------------------------------------------------- -# Helpers — DB node fixtures -# --------------------------------------------------------------------------- - - -def _make_db_node(n: int = 3) -> tuple[FunctionJobNode, InMemoryArrowDatabase, InMemoryArrowDatabase]: - """FunctionJobNode with persistent pipeline + result databases.""" - src = _make_source(n) - pod = FunctionPod( - PythonDataFunction(lambda value: value * 2, output_keys="result"), - ) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - node = FunctionJobNode( - pod, src, pipeline_database=pipeline_db, result_database=result_db - ) - return node, pipeline_db, result_db - - -# --------------------------------------------------------------------------- -# DB path — basic flow -# --------------------------------------------------------------------------- - - -class TestFunctionJobNodeAsyncExecuteDB: - - @pytest.mark.asyncio - async def test_all_items_processed_and_forwarded(self): - """All input items are computed and forwarded via the DB path.""" - node, pipeline_db, result_db = _make_db_node(3) - results = await _run_node(node) - assert len(results) == 3 - values = sorted(data.as_dict()["result"] for _, data in results) - assert values == [0, 2, 4] - - @pytest.mark.asyncio - async def test_pipeline_records_written_for_each_item(self): - """add_pipeline_record is called once per cache miss → one row per item.""" - node, pipeline_db, _ = _make_db_node(3) - await _run_node(node) - records = pipeline_db.get_all_records(node.node_identity_path) - assert records is not None - assert records.num_rows == 3 - - @pytest.mark.asyncio - async def test_correlation_key_absent_from_output_tags(self): - """Output tags must not contain the internal _tag_node_input_ref column.""" - node, _, _ = _make_db_node(3) - results = await _run_node(node) - full_key = f"{_constants.META_PREFIX}{_TAG_NODE_INPUT_REF}" - for tag, _ in results: - meta = tag.get_meta_info() - assert full_key not in meta, ( - f"Output tag leaked internal correlation key {full_key!r}" - ) - - @pytest.mark.asyncio - async def test_cache_hits_emitted_without_recomputation(self): - """Items already in the pipeline DB are served from cache (no recomputation).""" - node, _, _ = _make_db_node(3) - - # First run — populates both pipeline DB and result DB. - first_results = await _run_node(node) - assert len(first_results) == 3 - - # Reset in-memory state so async_execute re-runs from scratch. - node.clear_cache() - - call_count = 0 - original_process = node._function_pod.data_function.call - - def counted_call(data, **kw): - nonlocal call_count - call_count += 1 - return original_process(data, **kw) - - node._function_pod.data_function.call = counted_call # type: ignore[method-assign] - - second_results = await _run_node(node) - assert len(second_results) == 3 - # All 3 were hits — data function should NOT have been called again. - assert call_count == 0 - - @pytest.mark.asyncio - async def test_observer_cached_false_for_misses_cached_true_for_hits(self): - """Cache hits emit on_data_end(cached=True); misses emit on_data_end(cached=False).""" - node, _, _ = _make_db_node(2) - - spy = _SpyObserver() - - # First run — both are misses. - await _run_node(node) # populates DB - node.clear_cache() - - spy = _SpyObserver() - input_ch: Channel = Channel(buffer_size=8) - output_ch: Channel = Channel(buffer_size=8) - - async def feed(): - for tag, data in node._input_stream.iter_data(): - await input_ch.writer.send((tag, data)) - await input_ch.writer.close() - - await asyncio.gather( - feed(), - node.async_execute(input_ch.reader, output_ch.writer, observer=spy), - ) - await output_ch.reader.collect() - - # Second run was all hits. - assert len(spy.data_ends) == 2 - assert all(cached is True for _, _, _, _, cached in spy.data_ends) - - @pytest.mark.asyncio - async def test_semaphore_not_leaked_on_computation_crash(self): - """When a data function raises, processing continues for remaining items.""" - call_count = 0 - - def sometimes_fail(value: int) -> int: - nonlocal call_count - call_count += 1 - if value == 1: - raise ValueError("deliberate failure") - return value * 2 - - src = _make_source(3) - pod = FunctionPod( - PythonDataFunction(sometimes_fail, output_keys="result"), - pod_config=PodConfig(max_concurrency=1), - ) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - node = FunctionJobNode(pod, src, pipeline_database=pipeline_db, result_database=result_db) - - results = await _run_node(node) - # value=0 → 0, value=1 → crash (skipped), value=2 → 4 - assert len(results) == 2 - values = sorted(data.as_dict()["result"] for _, data in results) - assert values == [0, 4] - - -# --------------------------------------------------------------------------- -# DB path — ephemeral -# --------------------------------------------------------------------------- - - -class TestFunctionJobNodeAsyncExecuteEphemeral: - - @pytest.mark.asyncio - async def test_ephemeral_pipeline_records_have_is_ephemeral_true(self): - """Ephemeral execution writes pipeline records with is_ephemeral=True.""" - from orcapod.system_constants import constants - from orcapod.types import NodeConfig - - src = _make_source(2) - pod = FunctionPod( - PythonDataFunction(lambda value: value * 2, output_keys="result"), - ) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - ephemeral_db = InMemoryArrowDatabase() - node = FunctionJobNode( - pod, src, pipeline_database=pipeline_db, result_database=result_db - ) - node.node_config = NodeConfig(is_result_ephemeral=True) - node.set_ephemeral_store(ephemeral_db) - - results = await _run_node(node) - assert len(results) == 2 - - records = pipeline_db.get_all_records(node.node_identity_path) - assert records is not None - assert records.num_rows == 2 - # All rows should have is_ephemeral=True - is_eph_col = records.column(constants.IS_EPHEMERAL_COL).to_pylist() - assert all(v is True for v in is_eph_col) - - @pytest.mark.asyncio - async def test_ephemeral_missing_store_raises_runtime_error(self): - """Raising RuntimeError when is_result_ephemeral=True but no store set.""" - from orcapod.types import NodeConfig - - src = _make_source(1) - pod = FunctionPod( - PythonDataFunction(lambda value: value * 2, output_keys="result"), - ) - pipeline_db = InMemoryArrowDatabase() - result_db = InMemoryArrowDatabase() - node = FunctionJobNode( - pod, src, pipeline_database=pipeline_db, result_database=result_db - ) - node.node_config = NodeConfig(is_result_ephemeral=True) - # Deliberately do NOT call set_ephemeral_store() - - with pytest.raises(RuntimeError, match="ephemeral"): - await _run_node(node) -``` - -- [ ] **Step 2: Run the tests to confirm they fail** - -```bash -uv run pytest tests/test_core/nodes/test_function_job_node_async_execute.py::TestFunctionJobNodeAsyncExecuteDB \ - tests/test_core/nodes/test_function_job_node_async_execute.py::TestFunctionJobNodeAsyncExecuteEphemeral \ - -v -``` - -Expected: most or all tests FAIL because the 3-stage pipeline isn't implemented yet. -The test importing `_TAG_NODE_INPUT_REF` will cause an `ImportError` — that's expected. - -- [ ] **Step 3: Add the `_TAG_NODE_INPUT_REF` module constant to `function_node.py`** - -Near the top of `function_node.py` where the other private constants are defined (after -`_PIPELINE_RECOMPUTATION_INDEX_COL`), add: - -```python -# Private meta-column name stamped into tags routed to the compute channel. -# Carries the correlation key (bytes) that links an execution result back to -# its original (tag, input_data) pair. Stripped from output tags in -# record_and_forward() before downstream emission. -_TAG_NODE_INPUT_REF = "_tag_node_input_ref" -``` - -- [ ] **Step 4: Implement the 3-stage DB pipeline in `FunctionJobNode.async_execute()`** - -Find the DB path block in `async_execute()` (the `if self._cached_function_pod is not None:` branch, -starting around line 2103 after earlier edits). Replace the entire block with the 3-stage pipeline. - -The complete new `async_execute()` method body (replacing from after the UNAVAILABLE check to the -final `finally`) is: - -```python - from orcapod.pipeline.observer import NoOpObserver - - node_label = self.label - node_hash = self.content_hash().to_string() - - obs = observer if observer is not None else NoOpObserver() - ctx_obs = obs.contextualize(*self.node_identity_path) - - try: - tag_schema = self._input_stream.output_schema(columns={"system_tags": True})[0] - ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) - - if self._cached_function_pod is not None: - # ---------------------------------------------------------- - # DB path — 3-stage concurrent pipeline - # ---------------------------------------------------------- - is_ephemeral = bool(self._node_config.is_result_ephemeral) - if is_ephemeral: - if self._ephemeral_cached_pod is None: - raise RuntimeError( - f"FunctionJobNode '{self.label}' has is_result_ephemeral=True " - "but no ephemeral store has been assigned. Call " - "set_ephemeral_store() with an ArrowDatabaseProtocol before " - "executing this node." - ) - execution_pod = self._ephemeral_cached_pod - else: - execution_pod = self._cached_function_pod - - # Phase 1: load pipeline DB → in-memory cache keyed by base_entry_id. - loaded = self._load_cached_entries() - self._cached_output_datas.update(loaded) - if loaded: - self._cached_output_table = None - self._cached_content_hash_column = None - cached_by_base_entry_id: dict[bytes, tuple[TagProtocol, DataProtocol]] = dict(loaded) - - # Intermediate channels (bounded for backpressure). - compute_channel: Channel[tuple[TagProtocol, DataProtocol]] = Channel(buffer_size=16) - result_channel: Channel[tuple[TagProtocol, DataProtocol]] = Channel(buffer_size=16) - - # Local dict: correlation_key → (original_tag, original_input_data) - input_store: dict[bytes, tuple[TagProtocol, DataProtocol]] = {} - - async def route_inputs() -> None: - """Stage 1: send cache hits to output; stamp misses for computation.""" - try: - async for tag, data in input_channel: - base_entry_id = self.compute_base_entry_id(tag, data) - if base_entry_id in cached_by_base_entry_id: - cached_tag, cached_data = cached_by_base_entry_id[base_entry_id] - ctx_obs.on_data_start(node_label, tag, data) - ctx_obs.on_data_end( - node_label, tag, data, cached_data, cached=True - ) - await output.send((cached_tag, cached_data)) - else: - correlation_key = uuid.uuid4().bytes - input_store[correlation_key] = (tag, data) - stamped_tag = tag.with_meta_columns( - **{_TAG_NODE_INPUT_REF: correlation_key} - ) - await compute_channel.writer.send((stamped_tag, data)) - finally: - await compute_channel.writer.close() - - async def record_and_forward() -> None: - """Stage 3: record pipeline entry, strip key, emit to output.""" - async for output_tag, output_data in result_channel.reader: - correlation_key = output_tag.get_meta_value(_TAG_NODE_INPUT_REF) - original_tag, input_data = input_store.pop(correlation_key) - result_computed = bool( - output_data.get_meta_value( - execution_pod.RESULT_COMPUTED_FLAG, False - ) - ) - self.add_pipeline_record( - original_tag, - input_data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - is_ephemeral=is_ephemeral, - ) - # Update in-memory cache so iter_data() sees the result. - base_entry_id = self.compute_base_entry_id(original_tag, input_data) - clean_tag = output_tag.drop_meta_columns(_TAG_NODE_INPUT_REF) - self._cached_output_datas[base_entry_id] = (clean_tag, output_data) - self._cached_output_table = None - self._cached_content_hash_column = None - await output.send((clean_tag, output_data)) - - async with asyncio.TaskGroup() as tg: - tg.create_task(route_inputs()) - tg.create_task( - execution_pod.async_execute( - [compute_channel.reader], - result_channel.writer, - observer=ctx_obs, - ) - ) - tg.create_task(record_and_forward()) - - else: - # No-DB path: delegate dispatch entirely to the pod. - await self._function_pod.async_execute( - [input_channel], output, observer=ctx_obs - ) - - ctx_obs.on_node_end(node_label, node_hash) - finally: - await output.close() -``` - -**Important:** The `Channel` type needs to be imported in `function_node.py`. Verify the import -already exists (it should since it's in `orcapod.channels`): - -```python -from orcapod.channels import ReadableChannel, WritableChannel -``` - -You also need `Channel` itself for the intermediate channels. Add it: - -```python -from orcapod.channels import Channel, ReadableChannel, WritableChannel -``` - -- [ ] **Step 5: Run all tests** - -```bash -uv run pytest tests/ -v --timeout=30 -``` - -Expected: all tests pass, including all new DB and ephemeral tests. - -If `test_cache_hits_emitted_without_recomputation` fails, check that `clear_cache()` properly -resets `_cached_output_datas` (it should — it's an existing method). - -If `test_correlation_key_absent_from_output_tags` fails, check that `drop_meta_columns` is -called with the right key in `record_and_forward`. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py \ - tests/test_core/nodes/test_function_job_node_async_execute.py -git commit -m "feat(function_node): redesign async_execute as 3-stage concurrent pipeline - -FunctionJobNode.async_execute() now delegates computation to -execution_pod.async_execute() (CachedFunctionPod inheriting the loop -from _FunctionPodBase) via bounded compute_channel/result_channel. -Three asyncio.TaskGroup tasks run concurrently: - 1. route_inputs: cache hits → output, misses → compute_channel (stamped) - 2. execution_pod.async_execute: compute_channel → result_channel - 3. record_and_forward: add_pipeline_record, strip key, emit to output -Both persistent and ephemeral DB sub-paths use the same pipeline. -Adds _TAG_NODE_INPUT_REF module constant (private, never in public API). -Imports Channel for intermediate channels." -``` - ---- - -## Task 4: Cleanup — remove orphan test, relocate new observer tests - -The `TestAsyncExecuteBackpressure` in `test_regression_fixes.py` tested -`FunctionPod.async_execute()` in isolation — a method that was an orphan (never called by -the orchestrator). Its equivalent coverage now lives in `TestFunctionJobNodeAsyncExecuteNoDB.test_max_concurrency_limits_tasks`. Remove it. - -Also relocate the `TestFunctionPodBaseAsyncExecuteObserver` test added in Task 1 from -`test_regression_fixes.py` to its natural home (or keep in place — it tests pod-level observer -behavior, and `test_regression_fixes.py` is an acceptable home). Move it to the dedicated file -for cleanliness. - -**Files:** -- Modify: `tests/test_core/test_regression_fixes.py` -- Modify: `tests/test_core/nodes/test_function_job_node_async_execute.py` - -- [ ] **Step 1: Remove `TestAsyncExecuteBackpressure` from `test_regression_fixes.py`** - -Find and delete the entire `TestAsyncExecuteBackpressure` class (lines 293–343 in the original -file, may have shifted). It begins with: - -```python -class TestAsyncExecuteBackpressure: - """With max_concurrency set, pending tasks should be bounded.""" - ... -``` - -Delete from the class definition through the end of the last test in the class. - -Also remove the section header comment above it: -```python -# =========================================================================== -# 4. FunctionPod.async_execute backpressure bounds pending tasks -# =========================================================================== -``` - -Update the module-level docstring to remove item 4: - -```python -""" -Regression tests for bugs fixed in the data-function-executor-system branch. - -Covers: -1. async_execute output channel closed on exception (try/finally) -2. DataFunctionWrapper.direct_call/direct_async_call bypass executor routing -3. Concurrent iteration falls back to sequential inside a running event loop -4. _materialize_to_stream preserves source_info provenance tokens -5. RayExecutor._ensure_ray_initialized uses ray_address -6. DataFunctionExecutorProtocol uses DataFunctionProtocol (not Any) -""" -``` - -(Renumber items 5-7 → 4-6.) - -- [ ] **Step 2: Move `TestFunctionPodBaseAsyncExecuteObserver` to the dedicated test file** - -Remove `TestFunctionPodBaseAsyncExecuteObserver` from the bottom of `test_regression_fixes.py` -(added in Task 1). - -Add it to `tests/test_core/nodes/test_function_job_node_async_execute.py` — or optionally to -a separate `tests/test_core/function_pod/test_function_pod_base_async_execute.py`. Since these -tests test pod behavior (not node behavior), place them in a new file: - -Create `tests/test_core/function_pod/test_function_pod_async_execute.py`: - -```python -"""Tests for _FunctionPodBase.async_execute() — observer hooks and backpressure.""" - -from __future__ import annotations - -import asyncio - -import pyarrow as pa -import pytest - -from orcapod.channels import Channel -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.types import PodConfig - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_stream(n: int = 3) -> ArrowTableStream: - schema = pa.schema( - [pa.field("id", pa.int64(), nullable=False), pa.field("x", pa.int64(), nullable=False)] - ) - table = pa.table( - { - "id": pa.array(list(range(n)), type=pa.int64()), - "x": pa.array(list(range(n)), type=pa.int64()), - }, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -async def _feed(stream: ArrowTableStream, ch: Channel) -> None: - for tag, data in stream.iter_data(): - await ch.writer.send((tag, data)) - await ch.writer.close() - - -class _SpyObserver: - def __init__(self): - self.starts = [] - self.ends = [] # [(label, tag, input, output, cached)] - self.crashes = [] - - def contextualize(self, *path): - return self - - def on_node_start(self, *a, **kw): pass - def on_node_end(self, *a, **kw): pass - - def on_data_start(self, label, tag, data): - self.starts.append((label, tag, data)) - - def on_data_end(self, label, tag, inp, out, *, cached): - self.ends.append((label, tag, inp, out, cached)) - - def on_data_crash(self, label, tag, data, exc): - self.crashes.append(exc) - - def create_data_logger(self, tag, data): - return None - - -# --------------------------------------------------------------------------- -# Observer hooks -# --------------------------------------------------------------------------- - - -class TestFunctionPodBaseAsyncExecuteObserver: - @pytest.mark.asyncio - async def test_on_data_start_and_end_fire_per_item(self): - """on_data_start fires before and on_data_end(cached=False) after each item.""" - pf = PythonDataFunction(lambda x: x * 2, output_keys="result") - pod = FunctionPod(pf) - stream = _make_stream(3) - input_ch: Channel = Channel(buffer_size=16) - output_ch: Channel = Channel(buffer_size=16) - await _feed(stream, input_ch) - - spy = _SpyObserver() - await pod.async_execute([input_ch.reader], output_ch.writer, observer=spy) - results = await output_ch.reader.collect() - - assert len(results) == 3 - assert len(spy.starts) == 3 - assert len(spy.ends) == 3 - assert all(cached is False for _, _, _, _, cached in spy.ends) - - @pytest.mark.asyncio - async def test_on_data_crash_fires_not_on_data_end(self): - """on_data_crash fires (and on_data_end does not) when processing raises.""" - def boom(x: int) -> int: - raise ValueError("bang") - - pf = PythonDataFunction(boom, output_keys="result") - pod = FunctionPod(pf) - stream = _make_stream(2) - input_ch: Channel = Channel(buffer_size=16) - output_ch: Channel = Channel(buffer_size=16) - await _feed(stream, input_ch) - - spy = _SpyObserver() - await pod.async_execute([input_ch.reader], output_ch.writer, observer=spy) - results = await output_ch.reader.collect() - - assert len(results) == 0 - assert len(spy.crashes) == 2 - assert len(spy.ends) == 0 - - -# --------------------------------------------------------------------------- -# Backpressure -# --------------------------------------------------------------------------- - - -class TestFunctionPodBaseAsyncExecuteBackpressure: - @pytest.mark.asyncio - async def test_max_concurrency_limits_concurrent_tasks(self): - """With max_concurrency=1, at most one task runs concurrently.""" - concurrent_count = 0 - max_observed = 0 - - async def track(x: int) -> int: - nonlocal concurrent_count, max_observed - concurrent_count += 1 - max_observed = max(max_observed, concurrent_count) - await asyncio.sleep(0.01) - concurrent_count -= 1 - return x * 2 - - pf = PythonDataFunction(lambda x: x * 2, output_keys="result") - original_async_call = pf.async_call - - async def patched(data, **kw): - nonlocal concurrent_count, max_observed - concurrent_count += 1 - max_observed = max(max_observed, concurrent_count) - await asyncio.sleep(0.01) - concurrent_count -= 1 - return await original_async_call(data, **kw) - - pf.async_call = patched # type: ignore[method-assign] - - pod = FunctionPod(pf, pod_config=PodConfig(max_concurrency=1)) - stream = _make_stream(5) - input_ch: Channel = Channel(buffer_size=32) - output_ch: Channel = Channel(buffer_size=32) - await _feed(stream, input_ch) - await pod.async_execute([input_ch.reader], output_ch.writer) - results = await output_ch.reader.collect() - assert len(results) == 5 - assert max_observed <= 1 -``` - -- [ ] **Step 3: Run the full test suite** - -```bash -uv run pytest tests/ -v --timeout=30 -``` - -Expected: all tests pass. Verify: -- `TestAsyncExecuteBackpressure` is gone -- `TestFunctionPodBaseAsyncExecuteBackpressure` in the new file covers equivalent behavior -- All `TestFunctionJobNodeAsyncExecuteDB` and `TestFunctionJobNodeAsyncExecuteEphemeral` pass - -- [ ] **Step 4: Final import cleanup — verify `function_pod.py`** - -The observer types are used only in the `TYPE_CHECKING` block. Confirm the import is present -and correct: - -```bash -grep -n "ExecutionObserverProtocol" src/orcapod/core/function_pod.py -``` - -Expected: one line in the `TYPE_CHECKING` block. - -Also verify `PipelineConfig`, `PodConfig`, `resolve_concurrency` are still present in -`function_pod.py` (they ARE used there — `async_execute` on `_FunctionPodBase` still uses them): - -```bash -grep -n "PipelineConfig\|resolve_concurrency" src/orcapod/core/function_pod.py -``` - -Expected: lines referencing both in the import and in `async_execute`. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_core/test_regression_fixes.py \ - tests/test_core/nodes/test_function_job_node_async_execute.py \ - tests/test_core/function_pod/test_function_pod_async_execute.py -git commit -m "test(async_execute): remove orphan backpressure test, add dedicated coverage - -Removes TestAsyncExecuteBackpressure from test_regression_fixes.py -(tested an orphan FunctionPod method, never called by orchestrator). -Equivalent coverage lives in TestFunctionJobNodeAsyncExecuteNoDB and -TestFunctionPodBaseAsyncExecuteBackpressure. Adds: -- tests/test_core/function_pod/test_function_pod_async_execute.py: - observer hooks and backpressure via _FunctionPodBase.async_execute() -- tests/test_core/nodes/test_function_job_node_async_execute.py: - full FunctionJobNode.async_execute() coverage (no-DB, DB, ephemeral)" -``` - ---- - -## Self-Review - -### Spec coverage - -| Spec requirement | Task | -|---|---| -| `async_execute()` moves to `_FunctionPodBase` | Task 1 | -| `observer` parameter on `_FunctionPodBase.async_execute()` | Task 1 | -| `on_data_start`, `on_data_end`, `on_data_crash` per item | Task 1 | -| No-DB path delegates to `self._function_pod.async_execute()` | Task 2 | -| Delete orphan `_async_execute_one_data` | Task 2 | -| Dead import cleanup (PipelineConfig, PodConfig, resolve_concurrency) | Task 2 | -| `_TAG_NODE_INPUT_REF` private constant in `function_node.py` | Task 3 | -| 3-stage concurrent pipeline (route → compute → record) | Task 3 | -| route_inputs: cache hits → output, misses → compute_channel | Task 3 | -| record_and_forward: add_pipeline_record, strip key, emit | Task 3 | -| Bounded `compute_channel` + `result_channel` | Task 3 | -| Ephemeral path uses `_ephemeral_cached_pod`, `is_ephemeral=True` | Task 3 | -| RuntimeError guard when ephemeral=True but no store set | Task 3 | -| Output tags carry no `_tag_node_input_ref` | Task 3 (tested) | -| `add_pipeline_record` called once per cache miss | Task 3 (tested) | -| Remove `TestAsyncExecuteBackpressure` | Task 4 | -| New test file `test_function_job_node_async_execute.py` | Tasks 2–4 | -| CACHE_ONLY / UNAVAILABLE modes unchanged | ✓ preserved (no changes to those branches) | - -### Key invariants verification - -1. **Output tags carry no `_tag_node_input_ref`** — verified by `test_correlation_key_absent_from_output_tags` -2. **`add_pipeline_record` called once per cache miss** — verified by `test_pipeline_records_written_for_each_item` -3. **Cache hits bypass computation** — verified by `test_cache_hits_emitted_without_recomputation` -4. **`_FunctionPodBase.async_execute()` is unaware of pipeline DB** — architectural constraint, no DB imports in function_pod.py -5. **No-DB path uses `_function_pod`** — enforced by code structure (else branch) -6. **Ephemeral path uses `_ephemeral_cached_pod` + `is_ephemeral=True`** — verified by `test_ephemeral_pipeline_records_have_is_ephemeral_true` -7. **Persistent path uses `_cached_function_pod` + `is_ephemeral=False`** — verified by `test_pipeline_records_written_for_each_item` diff --git a/superpowers/plans/2026-07-10-itl-519-hash-cacher-write-guards.md b/superpowers/plans/2026-07-10-itl-519-hash-cacher-write-guards.md deleted file mode 100644 index a5668bdb1..000000000 --- a/superpowers/plans/2026-07-10-itl-519-hash-cacher-write-guards.md +++ /dev/null @@ -1,923 +0,0 @@ -# Hash Cacher Write Guards (ITL-519) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `read_only` and `min_cache_size_bytes` write guards to `InMemoryHashCacher` and `SqliteHashCacher`, expose them through `enable_file_hash_caching()`, and document them. - -**Architecture:** Both knobs live on the cachers and are enforced in `put()`. `get()` is unchanged. `CachedFileHasher` itself is not modified — it already calls `cacher.put()`, which the guard intercepts. `enable_file_hash_caching()` forwards the new kwargs directly to `SqliteHashCacher`. - -**Tech Stack:** Python 3.12, `uv run pytest`, `sqlite3`, `upath` - -**Spec:** `superpowers/specs/2026-07-10-itl-519-hash-cacher-write-guards-design.md` - -**Branch:** `eywalker/itl-519-filehasher-cache-add-read-only-mode-minimum-file-size` - ---- - -## File Map - -| File | Action | Responsibility | -|------|--------|---------------| -| `src/orcapod/hashing/hash_cachers.py` | Modify | Add `read_only`, `min_cache_size_bytes`, `__repr__` to both cachers | -| `src/orcapod/contexts/__init__.py` | Modify | Expose new kwargs in `enable_file_hash_caching()` | -| `tests/test_hashing/test_hash_cacher_write_guards.py` | Create | All write-guard tests for both cachers + `CachedFileHasher` integration | -| `tests/test_hashing/test_hash_cachers.py` | Modify | Add `enable_file_hash_caching()` pass-through tests | -| `docs/concepts/file-hash-caching.md` | Modify | Document both knobs with activation snippets | - ---- - -## Task 0: Check out the feature branch - -**Files:** none - -- [ ] **Step 0.1: Check out the branch** - -```bash -cd orcapod-python -git checkout main -git pull -git checkout -b eywalker/itl-519-filehasher-cache-add-read-only-mode-minimum-file-size -git branch --show-current -``` - -Expected output: `eywalker/itl-519-filehasher-cache-add-read-only-mode-minimum-file-size` - ---- - -## Task 1: Write failing tests for `InMemoryHashCacher` write guards - -Write the tests before touching any implementation. They will fail because -`InMemoryHashCacher` does not yet accept `read_only` or `min_cache_size_bytes`. - -**Files:** -- Create: `tests/test_hashing/test_hash_cacher_write_guards.py` - -- [ ] **Step 1.1: Create the test file** - -Create `tests/test_hashing/test_hash_cacher_write_guards.py` with this content: - -```python -"""Tests for hash cacher write guards: read_only and min_cache_size_bytes. - -Covers InMemoryHashCacher, SqliteHashCacher, CachedFileHasher integration, -and enable_file_hash_caching() pass-through — added by ITL-519. -""" - -from __future__ import annotations - -import pytest -from upath import UPath - -from orcapod.hashing.file_hashers import CachedFileHasher, FileHashKey, FileHasher -from orcapod.hashing.hash_cachers import InMemoryHashCacher, SqliteHashCacher -from orcapod.types import ContentHash - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def make_key( - path: str = "/a/b.txt", mtime_ns: int = 1000, size: int = 100 -) -> FileHashKey: - return FileHashKey(path=UPath(path), mtime_ns=mtime_ns, size=size) - - -def make_hash(digest: bytes = b"\xab" * 32) -> ContentHash: - return ContentHash(method="sha256", digest=digest) - - -# --------------------------------------------------------------------------- -# InMemoryHashCacher — read_only -# --------------------------------------------------------------------------- - - -class TestInMemoryHashCacherReadOnly: - def test_put_is_noop_when_read_only(self): - cacher = InMemoryHashCacher(read_only=True) - key = make_key() - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - def test_get_returns_none_on_empty_read_only_cacher(self): - cacher = InMemoryHashCacher(read_only=True) - assert cacher.get(make_key()) is None - - def test_put_works_when_not_read_only(self): - cacher = InMemoryHashCacher(read_only=False) - key = make_key() - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - def test_default_is_not_read_only(self): - cacher = InMemoryHashCacher() - key = make_key() - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - -# --------------------------------------------------------------------------- -# InMemoryHashCacher — min_cache_size_bytes -# --------------------------------------------------------------------------- - - -class TestInMemoryHashCacherThreshold: - def test_small_file_not_cached(self): - cacher = InMemoryHashCacher(min_cache_size_bytes=100) - key = make_key(size=50) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - def test_file_at_threshold_is_cached(self): - # Boundary is inclusive: size >= threshold → stored - cacher = InMemoryHashCacher(min_cache_size_bytes=100) - key = make_key(size=100) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - def test_large_file_is_cached(self): - cacher = InMemoryHashCacher(min_cache_size_bytes=100) - key = make_key(size=200) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - def test_none_threshold_caches_all(self): - cacher = InMemoryHashCacher(min_cache_size_bytes=None) - key = make_key(size=1) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - def test_zero_threshold_caches_all(self): - # 0 is falsy → treated as "no threshold" - cacher = InMemoryHashCacher(min_cache_size_bytes=0) - key = make_key(size=1) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - -# --------------------------------------------------------------------------- -# InMemoryHashCacher — combined -# --------------------------------------------------------------------------- - - -class TestInMemoryHashCacherCombined: - def test_read_only_wins_over_large_file(self): - cacher = InMemoryHashCacher(read_only=True, min_cache_size_bytes=100) - key = make_key(size=9999) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - def test_writable_large_file_is_cached(self): - cacher = InMemoryHashCacher(read_only=False, min_cache_size_bytes=100) - key = make_key(size=200) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) == value - - def test_writable_small_file_not_cached(self): - cacher = InMemoryHashCacher(read_only=False, min_cache_size_bytes=100) - key = make_key(size=50) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - -# --------------------------------------------------------------------------- -# InMemoryHashCacher — __repr__ -# --------------------------------------------------------------------------- - - -class TestInMemoryHashCacherRepr: - def test_repr_shows_read_only_false(self): - cacher = InMemoryHashCacher() - assert "read_only=False" in repr(cacher) - - def test_repr_shows_read_only_true(self): - cacher = InMemoryHashCacher(read_only=True) - assert "read_only=True" in repr(cacher) - - def test_repr_shows_min_cache_size_bytes_none(self): - cacher = InMemoryHashCacher() - assert "min_cache_size_bytes=None" in repr(cacher) - - def test_repr_shows_min_cache_size_bytes_value(self): - cacher = InMemoryHashCacher(min_cache_size_bytes=1024) - assert "min_cache_size_bytes=1024" in repr(cacher) - - -# --------------------------------------------------------------------------- -# SqliteHashCacher — read_only -# --------------------------------------------------------------------------- - - -class TestSqliteHashCacherReadOnly: - def test_put_is_noop_when_read_only(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", read_only=True) - key = make_key() - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - def test_read_only_cacher_can_read_preexisting_entries(self, tmp_path): - """A read-only SqliteHashCacher can still read entries written by a writable instance.""" - db = tmp_path / "cache.db" - key = make_key() - value = make_hash() - # Pre-populate with a writable cacher - writable = SqliteHashCacher(db) - writable.put(key, value) - writable.close() - # Read-only cacher must see the pre-existing entry - read_only = SqliteHashCacher(db, read_only=True) - result = read_only.get(key) - assert result is not None - assert result.digest == value.digest - - def test_put_works_when_not_read_only(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", read_only=False) - key = make_key() - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - def test_default_is_not_read_only(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db") - key = make_key() - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - -# --------------------------------------------------------------------------- -# SqliteHashCacher — min_cache_size_bytes -# --------------------------------------------------------------------------- - - -class TestSqliteHashCacherThreshold: - def test_small_file_not_cached(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", min_cache_size_bytes=100) - key = make_key(size=50) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - def test_file_at_threshold_is_cached(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", min_cache_size_bytes=100) - key = make_key(size=100) - value = make_hash() - cacher.put(key, value) - result = cacher.get(key) - assert result is not None - assert result.digest == value.digest - - def test_large_file_is_cached(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", min_cache_size_bytes=100) - key = make_key(size=200) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - def test_none_threshold_caches_all(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", min_cache_size_bytes=None) - key = make_key(size=1) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - def test_zero_threshold_caches_all(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", min_cache_size_bytes=0) - key = make_key(size=1) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - -# --------------------------------------------------------------------------- -# SqliteHashCacher — combined -# --------------------------------------------------------------------------- - - -class TestSqliteHashCacherCombined: - def test_read_only_wins_over_large_file(self, tmp_path): - cacher = SqliteHashCacher( - tmp_path / "cache.db", read_only=True, min_cache_size_bytes=100 - ) - key = make_key(size=9999) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - def test_writable_large_file_is_cached(self, tmp_path): - cacher = SqliteHashCacher( - tmp_path / "cache.db", read_only=False, min_cache_size_bytes=100 - ) - key = make_key(size=200) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - def test_writable_small_file_not_cached(self, tmp_path): - cacher = SqliteHashCacher( - tmp_path / "cache.db", read_only=False, min_cache_size_bytes=100 - ) - key = make_key(size=50) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - - -# --------------------------------------------------------------------------- -# SqliteHashCacher — __repr__ -# --------------------------------------------------------------------------- - - -class TestSqliteHashCacherRepr: - def test_repr_includes_db_path(self, tmp_path): - db = tmp_path / "cache.db" - cacher = SqliteHashCacher(db) - assert str(db) in repr(cacher) - - def test_repr_shows_read_only_false(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db") - assert "read_only=False" in repr(cacher) - - def test_repr_shows_read_only_true(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", read_only=True) - assert "read_only=True" in repr(cacher) - - def test_repr_shows_min_cache_size_bytes(self, tmp_path): - cacher = SqliteHashCacher(tmp_path / "cache.db", min_cache_size_bytes=1024) - assert "min_cache_size_bytes=1024" in repr(cacher) - - -# --------------------------------------------------------------------------- -# CachedFileHasher integration — read_only cacher (real files) -# --------------------------------------------------------------------------- - - -class TestCachedFileHasherWithReadOnlyCacher: - def test_hash_file_returns_correct_hash_despite_read_only(self, tmp_path): - """hash_file() computes and returns the hash even when the cacher is read-only.""" - f = tmp_path / "file.txt" - f.write_text("hello world") - - inner = FileHasher(algorithm="sha256") - cacher = InMemoryHashCacher(read_only=True) - cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - - result = cached.hash_file(f) - expected = inner.hash_file(f) - - assert result.method == expected.method - assert result.digest == expected.digest - - def test_hash_file_does_not_store_result_in_read_only_cacher(self, tmp_path): - """hash_file() returns the correct hash but stores nothing in the cacher.""" - f = tmp_path / "file.txt" - f.write_text("hello world") - - inner = FileHasher(algorithm="sha256") - cacher = InMemoryHashCacher(read_only=True) - cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - - cached.hash_file(f) - - path = UPath(f).resolve() - stat = path.stat() - key = FileHashKey(path, stat.st_mtime_ns, stat.st_size) - assert cacher.get(key) is None - - -# --------------------------------------------------------------------------- -# CachedFileHasher integration — threshold cacher (real files) -# --------------------------------------------------------------------------- - - -class TestCachedFileHasherWithThresholdCacher: - def test_small_file_hashed_and_returned_but_not_stored(self, tmp_path): - """Files below the threshold: hash is computed and returned, but not stored.""" - f = tmp_path / "small.bin" - f.write_bytes(b"x" * 10) # 10 bytes — below threshold of 100 - - inner = FileHasher(algorithm="sha256") - cacher = InMemoryHashCacher(min_cache_size_bytes=100) - cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - - result = cached.hash_file(f) - expected = inner.hash_file(f) - - assert result.digest == expected.digest # correct hash returned - - path = UPath(f).resolve() - stat = path.stat() - key = FileHashKey(path, stat.st_mtime_ns, stat.st_size) - assert cacher.get(key) is None # nothing stored - - def test_large_file_hashed_and_stored(self, tmp_path): - """Files at or above the threshold: hash is computed, returned, and stored.""" - f = tmp_path / "large.bin" - f.write_bytes(b"x" * 200) # 200 bytes — at/above threshold of 100 - - inner = FileHasher(algorithm="sha256") - cacher = InMemoryHashCacher(min_cache_size_bytes=100) - cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - - result = cached.hash_file(f) - - path = UPath(f).resolve() - stat = path.stat() - key = FileHashKey(path, stat.st_mtime_ns, stat.st_size) - stored = cacher.get(key) - assert stored is not None - assert stored.digest == result.digest -``` - -- [ ] **Step 1.2: Run the new tests to confirm they all fail** - -```bash -uv run pytest tests/test_hashing/test_hash_cacher_write_guards.py -v 2>&1 | head -60 -``` - -Expected: Many `TypeError: __init__() got an unexpected keyword argument 'read_only'` errors. Every test should fail — if any pass, investigate before continuing. - ---- - -## Task 2: Implement `InMemoryHashCacher` write guards + `__repr__` - -**Files:** -- Modify: `src/orcapod/hashing/hash_cachers.py` - -- [ ] **Step 2.1: Update `InMemoryHashCacher`** - -Open `src/orcapod/hashing/hash_cachers.py`. Replace the entire `InMemoryHashCacher` class with: - -```python -class InMemoryHashCacher: - """Dict-backed file hash cacher for testing and ephemeral in-process use. - - No persistence, no thread-safety guarantees beyond the GIL, no eviction. - Use ``SqliteHashCacher`` for production workloads. - - Args: - read_only: When ``True``, all ``put()`` calls are silent no-ops. - ``get()`` still works normally. Defaults to ``False``. - min_cache_size_bytes: When set to a positive integer, files whose - ``key.size`` is strictly below this threshold are not inserted. - ``None`` and ``0`` disable the threshold (default behaviour). - Defaults to ``None``. - """ - - def __init__( - self, - *, - read_only: bool = False, - min_cache_size_bytes: int | None = None, - ) -> None: - self._cache: dict[FileHashKey, ContentHash] = {} - self._read_only = read_only - self._min_cache_size_bytes = min_cache_size_bytes - - def get(self, key: FileHashKey) -> ContentHash | None: - """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. - - Args: - key: File hash cache key. - - Returns: - Cached ``ContentHash``, or ``None`` if not found. - """ - return self._cache.get(key) - - def put(self, key: FileHashKey, value: ContentHash) -> None: - """Store ``value`` under ``key``. - - No-ops silently when ``read_only=True`` or when ``key.size`` is below - ``min_cache_size_bytes``. - - Args: - key: File hash cache key. - value: ``ContentHash`` to store. - """ - if self._read_only: - return - if self._min_cache_size_bytes and key.size < self._min_cache_size_bytes: - return - self._cache[key] = value - - def clear(self) -> None: - """Remove all entries from the cache.""" - self._cache.clear() - - def __repr__(self) -> str: - return ( - f"InMemoryHashCacher(" - f"read_only={self._read_only!r}, " - f"min_cache_size_bytes={self._min_cache_size_bytes!r})" - ) -``` - -- [ ] **Step 2.2: Run only the `InMemoryHashCacher` tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cacher_write_guards.py -k "InMemory" -v -``` - -Expected: All `TestInMemoryHashCacher*` tests pass. The `TestSqliteHashCacher*` and `TestCachedFileHasher*` tests still fail — that is expected at this point. - -- [ ] **Step 2.3: Run existing InMemoryHashCacher tests to confirm no regression** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestInMemoryHashCacher -v -``` - -Expected: All existing tests pass. - -- [ ] **Step 2.4: Commit** - -```bash -git add src/orcapod/hashing/hash_cachers.py tests/test_hashing/test_hash_cacher_write_guards.py -git commit -m "feat(hashing): add read_only and min_cache_size_bytes guards to InMemoryHashCacher" -``` - ---- - -## Task 3: Implement `SqliteHashCacher` write guards + `__repr__` - -**Files:** -- Modify: `src/orcapod/hashing/hash_cachers.py` - -- [ ] **Step 3.1: Update `SqliteHashCacher.__init__`** - -In `src/orcapod/hashing/hash_cachers.py`, replace the `SqliteHashCacher.__init__` signature and body with: - -```python - def __init__( - self, - db_path: Path | None = None, - *, - read_only: bool = False, - min_cache_size_bytes: int | None = None, - ) -> None: - self.db_path = Path( - db_path - or os.environ.get("ORCAPOD_HASH_CACHE_DB") - or self.DEFAULT_DB_PATH - ) - self._read_only = read_only - self._min_cache_size_bytes = min_cache_size_bytes - self._local = threading.local() - self._ensure_schema() -``` - -- [ ] **Step 3.2: Update `SqliteHashCacher.put`** - -Replace the existing `put` method with: - -```python - def put(self, key: FileHashKey, value: ContentHash) -> None: - """Store ``value`` under ``key``. - - No-ops silently when ``read_only=True`` or when ``key.size`` is below - ``min_cache_size_bytes``. Uses ``INSERT OR REPLACE`` so writes are - idempotent when they do proceed. - - Args: - key: File hash cache key. - value: ``ContentHash`` to store. - """ - if self._read_only: - return - if self._min_cache_size_bytes and key.size < self._min_cache_size_bytes: - return - conn = self._connection() - conn.execute( - """ - INSERT OR REPLACE INTO file_hash_cache (path, mtime_ns, size, hash) - VALUES (?, ?, ?, ?) - """, - (str(key.path), key.mtime_ns, key.size, value.to_prefixed_digest()), - ) - conn.commit() -``` - -- [ ] **Step 3.3: Add `__repr__` to `SqliteHashCacher`** - -Add this method to `SqliteHashCacher` (before `__enter__`): - -```python - def __repr__(self) -> str: - return ( - f"SqliteHashCacher(" - f"db_path={str(self.db_path)!r}, " - f"read_only={self._read_only!r}, " - f"min_cache_size_bytes={self._min_cache_size_bytes!r})" - ) -``` - -Also update the class docstring to document the new args. Replace the existing docstring with: - -```python - """SQLite-backed file hash cacher. - - Stores file hashes keyed on ``(path, mtime_ns, size)`` in a local - SQLite database. Uses WAL mode for single-writer/multi-reader - concurrency and thread-local connections for thread safety. - - The hash is stored as a BLOB in ``{method}:{raw_digest}`` format via - ``ContentHash.to_prefixed_digest()``. - - Args: - db_path: Path to the SQLite database file. Defaults to - ``~/.orcapod/file_hash_cache.db`` or the - ``ORCAPOD_HASH_CACHE_DB`` environment variable. - read_only: When ``True``, all ``put()`` calls are silent no-ops. - ``get()`` still works normally. Defaults to ``False``. - min_cache_size_bytes: When set to a positive integer, files whose - ``key.size`` is strictly below this threshold are not inserted. - ``None`` and ``0`` disable the threshold (default behaviour). - Defaults to ``None``. - - Note: - Heavy multi-writer scenarios are a known SQLite limitation. A Turso - / libSQL migration is planned as a follow-up issue. - """ -``` - -- [ ] **Step 3.4: Run all write-guard tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cacher_write_guards.py -v -``` - -Expected: All tests pass. - -- [ ] **Step 3.5: Run existing SqliteHashCacher tests to confirm no regression** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestSqliteHashCacher -v -``` - -Expected: All existing tests pass. - -- [ ] **Step 3.6: Commit** - -```bash -git add src/orcapod/hashing/hash_cachers.py -git commit -m "feat(hashing): add read_only and min_cache_size_bytes guards to SqliteHashCacher" -``` - ---- - -## Task 4: Add `enable_file_hash_caching()` pass-through tests and implementation - -**Files:** -- Modify: `tests/test_hashing/test_hash_cachers.py` -- Modify: `src/orcapod/contexts/__init__.py` - -- [ ] **Step 4.1: Add tests to `TestEnableFileHashCaching`** - -Open `tests/test_hashing/test_hash_cachers.py`. Inside the `TestEnableFileHashCaching` class, add these two methods at the end: - -```python - def test_read_only_kwarg_passes_through_to_cacher( - self, restore_default_file_handler, tmp_path - ): - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - from orcapod.hashing.hash_cachers import SqliteHashCacher - - enable_file_hash_caching(db_path=tmp_path / "cache.db", read_only=True) - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - cacher = handler.file_hasher.cacher - - assert isinstance(cacher, SqliteHashCacher) - assert cacher._read_only is True - - def test_min_cache_size_bytes_kwarg_passes_through_to_cacher( - self, restore_default_file_handler, tmp_path - ): - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - from orcapod.hashing.hash_cachers import SqliteHashCacher - - enable_file_hash_caching( - db_path=tmp_path / "cache.db", min_cache_size_bytes=1024 - ) - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - cacher = handler.file_hasher.cacher - - assert isinstance(cacher, SqliteHashCacher) - assert cacher._min_cache_size_bytes == 1024 -``` - -- [ ] **Step 4.2: Run the new tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching::test_read_only_kwarg_passes_through_to_cacher tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching::test_min_cache_size_bytes_kwarg_passes_through_to_cacher -v -``` - -Expected: Both tests fail with `TypeError: enable_file_hash_caching() got an unexpected keyword argument 'read_only'`. - -- [ ] **Step 4.3: Update `enable_file_hash_caching()` signature and body** - -Open `src/orcapod/contexts/__init__.py`. Replace the function signature line: - -```python -def enable_file_hash_caching(db_path: "Path | None" = None) -> None: -``` - -with: - -```python -def enable_file_hash_caching( - db_path: "Path | None" = None, - read_only: bool = False, - min_cache_size_bytes: int | None = None, -) -> None: -``` - -Also update the docstring to document the new args. After the `db_path` arg in the `Args:` section, add: - -``` - read_only: When ``True``, the underlying ``SqliteHashCacher`` will - not insert new entries. Lookups still work normally. Defaults - to ``False``. - min_cache_size_bytes: When set, files smaller than this byte count - are not inserted into the cache. ``None`` and ``0`` disable the - threshold. Defaults to ``None``. -``` - -Then find the `SqliteHashCacher(db_path)` call inside the function body and replace it with: - -```python - cached_file_hasher = CachedFileHasher( - file_hasher=base_hasher, - cacher=SqliteHashCacher( - db_path, - read_only=read_only, - min_cache_size_bytes=min_cache_size_bytes, - ), - ) -``` - -- [ ] **Step 4.4: Run the pass-through tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching -v -``` - -Expected: All `TestEnableFileHashCaching` tests pass, including the two new ones. - -- [ ] **Step 4.5: Run the full test suite for hash cachers** - -```bash -uv run pytest tests/test_hashing/ -v -``` - -Expected: All tests pass. - -- [ ] **Step 4.6: Commit** - -```bash -git add tests/test_hashing/test_hash_cachers.py src/orcapod/contexts/__init__.py -git commit -m "feat(contexts): expose read_only and min_cache_size_bytes in enable_file_hash_caching" -``` - ---- - -## Task 5: Update documentation - -**Files:** -- Modify: `docs/concepts/file-hash-caching.md` - -- [ ] **Step 5.1: Add the "Controlling when the cache is written" section** - -Open `docs/concepts/file-hash-caching.md`. Find the line: - -```markdown -## Directory hashing (op.Directory) -``` - -Insert the following new section immediately before it (with a blank line above): - -```markdown -## Controlling when the cache is written - -By default, every file that passes through ``CachedFileHasher`` is inserted -into the cache on a miss. Two optional knobs let you restrict this. - -### Read-only mode - -Use ``read_only=True`` when you want lookups from a shared or authoritative -cache but must not add new entries to it — for example, when consuming a -cache pre-populated by ``populate_hash_cache()`` without polluting it with -ad-hoc entries. - -```python -import orcapod as op - -op.enable_file_hash_caching(db_path="/shared/cache.db", read_only=True) -``` - -Cache hits still work normally. On a miss, the file is hashed directly and -the result is returned to the caller — but it is never written to the cache. - -### Minimum file size threshold - -Use ``min_cache_size_bytes`` to skip the cache write overhead for small -files. For small files, the disk I/O bottleneck does not apply, so the -cache lookup and write add latency without meaningful savings. - -```python -import orcapod as op - -# Skip caching for files smaller than 1 MB -op.enable_file_hash_caching(min_cache_size_bytes=1_048_576) -``` - -Files smaller than the threshold are still hashed and the hash is returned -to the caller — they are simply not inserted into the cache. Files at or -above the threshold behave normally. Set to ``None`` or ``0`` to disable -the threshold (the default). - -### Combining both - -The two knobs compose independently. ``read_only=True`` takes precedence: -when enabled, no entry is ever written regardless of file size. -``min_cache_size_bytes`` is an additional guard that applies only when the -cacher is writable. - -```python -import orcapod as op - -# Read-only + skip files below 512 KB (threshold is moot when read-only, -# but harmless and documents intent) -op.enable_file_hash_caching( - db_path="/shared/cache.db", - read_only=True, - min_cache_size_bytes=524_288, -) -``` - -``` - -- [ ] **Step 5.2: Update the "When caching helps" section** - -In the same file, find the bullet that begins: - -```markdown -- Files are small. -``` - -Replace it with: - -```markdown -- Files are small. Disk I/O is not the bottleneck for small files, so the - cache lookup adds overhead without meaningful savings. Use - ``min_cache_size_bytes`` to skip caching small files automatically. -``` - -If the bullet does not already exist, add it to the "Caching does not help much when" list. - -- [ ] **Step 5.3: Verify the docs file reads cleanly** - -```bash -cat docs/concepts/file-hash-caching.md -``` - -Scan for any broken Markdown (unclosed code fences, stray backticks). Fix any formatting issues. - -- [ ] **Step 5.4: Commit** - -```bash -git add docs/concepts/file-hash-caching.md -git commit -m "docs(hashing): document read_only and min_cache_size_bytes knobs" -``` - ---- - -## Task 6: Final check — run the complete hashing test suite - -- [ ] **Step 6.1: Run all hashing tests** - -```bash -uv run pytest tests/test_hashing/ -v -``` - -Expected: All tests pass with no warnings about unexpected kwargs or missing attributes. - -- [ ] **Step 6.2: Push the branch** - -```bash -git push -u origin eywalker/itl-519-filehasher-cache-add-read-only-mode-minimum-file-size -``` - -Expected: Branch pushed, no errors. diff --git a/superpowers/plans/2026-07-10-itl-520-postgres-hash-cacher.md b/superpowers/plans/2026-07-10-itl-520-postgres-hash-cacher.md deleted file mode 100644 index 83cebcb6f..000000000 --- a/superpowers/plans/2026-07-10-itl-520-postgres-hash-cacher.md +++ /dev/null @@ -1,1016 +0,0 @@ -# PostgreSQL Hash Cacher (ITL-520) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `PostgresHashCacher` — a Postgres-backed drop-in alternative to `SqliteHashCacher` — plus a `conninfo` parameter on `enable_file_hash_caching()` to activate it. - -**Architecture:** `PostgresHashCacher` lives in its own module (`postgres_hash_cacher.py`), uses thread-local psycopg3 connections, and stores hashes in a `file_hash_cache` table with `INSERT ... ON CONFLICT DO NOTHING` for concurrency safety. It is exported from `orcapod.hashing` under a `try/except ImportError` guard so users without `psycopg` installed pay nothing. Tests use `testcontainers.postgres.PostgresContainer` against a real Postgres. - -**Tech Stack:** Python 3.12, `psycopg[binary]>=3.0`, `testcontainers[postgres]>=4.0.0`, `uv run pytest` - -**Spec:** `superpowers/specs/2026-07-10-itl-520-postgres-hash-cacher-design.md` - -**Branch:** `eywalker/itl-520-filehasher-add-postgresql-backend-for-the-hash-cacher` - ---- - -## File Map - -| File | Action | Responsibility | -|------|--------|----------------| -| `src/orcapod/hashing/postgres_hash_cacher.py` | Create | `PostgresHashCacher` — full implementation | -| `tests/test_hashing/test_postgres_hash_cacher.py` | Create | All `PostgresHashCacher` tests via testcontainers | -| `src/orcapod/hashing/__init__.py` | Modify | Export `PostgresHashCacher` under `try/except ImportError` | -| `src/orcapod/contexts/__init__.py` | Modify | Add `conninfo` param to `enable_file_hash_caching()` | -| `tests/test_hashing/test_hash_cachers.py` | Modify | Add `ValueError` test for mutually-exclusive `conninfo`/`db_path` | -| `pyproject.toml` | Modify | Add `testcontainers[postgres]>=4.0.0` to dev deps | -| `docs/concepts/file-hash-caching.md` | Modify | Add "Choosing a backend: SQLite vs Postgres" section | - ---- - -## Task 0: Check out the feature branch - -**Files:** none - -- [ ] **Step 0.1: Check out the branch** - -```bash -git checkout main -git pull -git checkout -b eywalker/itl-520-filehasher-add-postgresql-backend-for-the-hash-cacher -git branch --show-current -``` - -Expected output: `eywalker/itl-520-filehasher-add-postgresql-backend-for-the-hash-cacher` - ---- - -## Task 1: Add `testcontainers[postgres]` to dev deps - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1.1: Add the dev dependency** - -Open `pyproject.toml`. In the `[dependency-groups]` → `dev` list, add `testcontainers[postgres]` immediately after the existing `testcontainers[minio]` line: - -```toml - "testcontainers[minio]>=4.0.0", - "testcontainers[postgres]>=4.0.0", -``` - -- [ ] **Step 1.2: Sync and verify** - -```bash -uv sync -uv run python -c "from testcontainers.postgres import PostgresContainer; print('OK')" -``` - -Expected: `OK` - -- [ ] **Step 1.3: Commit** - -```bash -git add pyproject.toml uv.lock -git commit -m "chore(deps): add testcontainers[postgres] to dev deps (ITL-520)" -``` - ---- - -## Task 2: Write failing tests for `PostgresHashCacher` basic operations - -Write the tests first. They will fail with `ImportError` / `ModuleNotFoundError` because `postgres_hash_cacher.py` does not exist yet. - -**Files:** -- Create: `tests/test_hashing/test_postgres_hash_cacher.py` - -- [ ] **Step 2.1: Create the test file** - -Create `tests/test_hashing/test_postgres_hash_cacher.py` with this content: - -```python -"""Tests for PostgresHashCacher using a real Postgres via testcontainers.""" - -from __future__ import annotations - -import threading - -import pytest -from upath import UPath - -psycopg = pytest.importorskip("psycopg") - -from testcontainers.postgres import PostgresContainer # noqa: E402 - -from orcapod.hashing.file_hashers import FileHashKey # noqa: E402 -from orcapod.hashing.postgres_hash_cacher import PostgresHashCacher # noqa: E402 -from orcapod.types import ContentHash # noqa: E402 - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def make_key( - path: str = "/a/b.txt", mtime_ns: int = 1000, size: int = 100 -) -> FileHashKey: - return FileHashKey(path=UPath(path), mtime_ns=mtime_ns, size=size) - - -def make_hash(digest: bytes = b"\xab" * 32) -> ContentHash: - return ContentHash(method="sha256", digest=digest) - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope="module") -def pg_conninfo(): - """Start one Postgres container for the entire module; yield a conninfo string.""" - with PostgresContainer("postgres:16") as pg: - yield ( - f"host={pg.get_container_host_ip()} " - f"port={pg.get_exposed_port(5432)} " - f"dbname={pg.dbname} " - f"user={pg.username} " - f"password={pg.password}" - ) - - -@pytest.fixture() -def cacher(pg_conninfo): - """Fresh PostgresHashCacher with a cleared table for each test.""" - c = PostgresHashCacher(pg_conninfo) - c.clear() - yield c - c.close() - - -# --------------------------------------------------------------------------- -# Basic operations -# --------------------------------------------------------------------------- - - -@pytest.mark.postgres -class TestPostgresHashCacherBasic: - def test_miss_returns_none(self, cacher): - assert cacher.get(make_key()) is None - - def test_put_then_get_returns_hit(self, cacher): - key = make_key() - value = make_hash() - cacher.put(key, value) - result = cacher.get(key) - assert result is not None - assert result.method == value.method - assert result.digest == value.digest - - def test_different_path_is_miss(self, cacher): - cacher.put(make_key("/a/b.txt"), make_hash(b"\xaa" * 32)) - assert cacher.get(make_key("/a/c.txt")) is None - - def test_different_mtime_ns_is_miss(self, cacher): - cacher.put(make_key(mtime_ns=1000), make_hash(b"\xaa" * 32)) - assert cacher.get(make_key(mtime_ns=2000)) is None - - def test_different_size_is_miss(self, cacher): - cacher.put(make_key(size=100), make_hash(b"\xaa" * 32)) - assert cacher.get(make_key(size=200)) is None - - def test_clear_empties_table(self, cacher): - key = make_key() - cacher.put(key, make_hash()) - cacher.clear() - assert cacher.get(key) is None - - def test_persistence_across_instances(self, pg_conninfo): - key = make_key("/persist.txt", mtime_ns=9999, size=512) - value = make_hash(b"\xcc" * 32) - - cacher1 = PostgresHashCacher(pg_conninfo) - cacher1.clear() - cacher1.put(key, value) - cacher1.close() - - cacher2 = PostgresHashCacher(pg_conninfo) - result = cacher2.get(key) - cacher2.close() - - assert result is not None - assert result.method == value.method - assert result.digest == value.digest - - def test_context_manager_closes_connection(self, pg_conninfo): - key = make_key("/ctx.txt", mtime_ns=7777, size=64) - value = make_hash(b"\xdd" * 32) - - with PostgresHashCacher(pg_conninfo) as c: - c.clear() - c.put(key, value) - - # Verify persistence (connection was closed but data committed) - cacher2 = PostgresHashCacher(pg_conninfo) - result = cacher2.get(key) - cacher2.close() - assert result is not None - assert result.digest == value.digest - - -# --------------------------------------------------------------------------- -# Write guards -# --------------------------------------------------------------------------- - - -@pytest.mark.postgres -class TestPostgresHashCacherReadOnly: - def test_put_is_noop_when_read_only(self, pg_conninfo): - key = make_key("/ro.txt") - value = make_hash() - - cacher = PostgresHashCacher(pg_conninfo, read_only=True) - cacher.clear() - cacher.put(key, value) - assert cacher.get(key) is None - cacher.close() - - def test_read_only_can_read_preexisting(self, pg_conninfo): - key = make_key("/ro_pre.txt", mtime_ns=5555, size=256) - value = make_hash(b"\xee" * 32) - - writable = PostgresHashCacher(pg_conninfo) - writable.clear() - writable.put(key, value) - writable.close() - - read_only = PostgresHashCacher(pg_conninfo, read_only=True) - result = read_only.get(key) - read_only.close() - - assert result is not None - assert result.digest == value.digest - - def test_default_is_not_read_only(self, cacher): - key = make_key() - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - - -@pytest.mark.postgres -class TestPostgresHashCacherThreshold: - def test_small_file_not_cached(self, pg_conninfo): - cacher = PostgresHashCacher(pg_conninfo, min_cache_size_bytes=100) - cacher.clear() - key = make_key(size=50) - cacher.put(key, make_hash()) - assert cacher.get(key) is None - cacher.close() - - def test_file_at_threshold_is_cached(self, pg_conninfo): - cacher = PostgresHashCacher(pg_conninfo, min_cache_size_bytes=100) - cacher.clear() - key = make_key(size=100) - value = make_hash() - cacher.put(key, value) - result = cacher.get(key) - cacher.close() - assert result is not None - assert result.digest == value.digest - - def test_large_file_is_cached(self, pg_conninfo): - cacher = PostgresHashCacher(pg_conninfo, min_cache_size_bytes=100) - cacher.clear() - key = make_key(size=200) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - cacher.close() - - def test_none_threshold_caches_all(self, pg_conninfo): - cacher = PostgresHashCacher(pg_conninfo, min_cache_size_bytes=None) - cacher.clear() - key = make_key(size=1) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - cacher.close() - - def test_zero_threshold_caches_all(self, pg_conninfo): - cacher = PostgresHashCacher(pg_conninfo, min_cache_size_bytes=0) - cacher.clear() - key = make_key(size=1) - value = make_hash() - cacher.put(key, value) - assert cacher.get(key) is not None - cacher.close() - - -# --------------------------------------------------------------------------- -# Concurrency -# --------------------------------------------------------------------------- - - -@pytest.mark.postgres -class TestPostgresHashCacherConcurrency: - def test_concurrent_insert_no_error_one_row(self, pg_conninfo): - """Two threads inserting the same key simultaneously → no error, one row.""" - key = make_key("/concurrent.txt", mtime_ns=12345, size=999) - value = make_hash(b"\xff" * 32) - - # Pre-clear using a dedicated cacher - setup = PostgresHashCacher(pg_conninfo) - setup.clear() - setup.close() - - errors: list[Exception] = [] - - def insert() -> None: - try: - c = PostgresHashCacher(pg_conninfo) - c.put(key, value) - c.close() - except Exception as exc: - errors.append(exc) - - threads = [threading.Thread(target=insert) for _ in range(2)] - for t in threads: - t.start() - for t in threads: - t.join() - - assert errors == [], f"Concurrent inserts raised errors: {errors}" - - # Verify exactly one row was written - with psycopg.connect(pg_conninfo) as conn: - count = conn.execute( - "SELECT COUNT(*) FROM file_hash_cache " - "WHERE path=%s AND mtime_ns=%s AND size=%s", - (str(key.path), key.mtime_ns, key.size), - ).fetchone()[0] - assert count == 1 - - -# --------------------------------------------------------------------------- -# Parity with SqliteHashCacher -# --------------------------------------------------------------------------- - - -@pytest.mark.postgres -class TestPostgresHashCacherParity: - def test_parity_with_sqlite(self, pg_conninfo, tmp_path): - """Same key/value → equivalent ContentHash from both backends.""" - from orcapod.hashing.hash_cachers import SqliteHashCacher - - key = make_key("/parity.txt", mtime_ns=111, size=222) - value = make_hash(b"\x12" * 32) - - pg_cacher = PostgresHashCacher(pg_conninfo) - pg_cacher.clear() - pg_cacher.put(key, value) - pg_result = pg_cacher.get(key) - pg_cacher.close() - - sq_cacher = SqliteHashCacher(tmp_path / "parity.db") - sq_cacher.put(key, value) - sq_result = sq_cacher.get(key) - - assert pg_result is not None - assert sq_result is not None - assert pg_result.method == sq_result.method - assert pg_result.digest == sq_result.digest - - -# --------------------------------------------------------------------------- -# Validation -# --------------------------------------------------------------------------- - - -@pytest.mark.postgres -class TestPostgresHashCacherValidation: - def test_negative_min_cache_size_bytes_raises(self, pg_conninfo): - with pytest.raises(ValueError, match="non-negative"): - PostgresHashCacher(pg_conninfo, min_cache_size_bytes=-1) - - def test_repr_includes_conninfo_and_guards(self, pg_conninfo): - cacher = PostgresHashCacher( - pg_conninfo, read_only=True, min_cache_size_bytes=1024 - ) - r = repr(cacher) - assert "PostgresHashCacher" in r - assert "read_only=True" in r - assert "min_cache_size_bytes=1024" in r - cacher.close() -``` - -- [ ] **Step 2.2: Run tests to verify they fail at import** - -```bash -uv run pytest tests/test_hashing/test_postgres_hash_cacher.py -v 2>&1 | head -20 -``` - -Expected: error containing `ModuleNotFoundError` or `ImportError: cannot import name 'PostgresHashCacher'` - ---- - -## Task 3: Implement `PostgresHashCacher` - -**Files:** -- Create: `src/orcapod/hashing/postgres_hash_cacher.py` - -- [ ] **Step 3.1: Create the implementation file** - -Create `src/orcapod/hashing/postgres_hash_cacher.py` with this content: - -```python -"""PostgreSQL-backed file hash cacher. - -Provides ``PostgresHashCacher`` — a network-accessible, concurrent-safe -implementation of ``CacherProtocol[FileHashKey, ContentHash]``. Requires -the optional ``psycopg`` driver (``pip install 'orcapod[postgresql]'``). -""" - -from __future__ import annotations - -import threading -from typing import TYPE_CHECKING - -from orcapod.hashing.file_hashers import FileHashKey -from orcapod.types import ContentHash - -if TYPE_CHECKING: - import psycopg as _psycopg - - -class PostgresHashCacher: - """PostgreSQL-backed file hash cacher. - - Stores file hashes keyed on ``(path, mtime_ns, size)`` in a shared - PostgreSQL database. Uses thread-local connections for thread safety - and ``INSERT ... ON CONFLICT DO NOTHING`` for concurrent-insert safety. - - The hash is stored as a BYTEA in ``{method}:{raw_digest}`` format via - ``ContentHash.to_prefixed_digest()``. - - Requires ``psycopg[binary]>=3.0`` (install with - ``pip install 'orcapod[postgresql]'``). - - Args: - conninfo: psycopg3 connection string, e.g. - ``"postgresql://user:pass@host:5432/dbname"`` or keyword DSN - ``"host=myhost dbname=mydb user=myuser password=mypass"``. - read_only: When ``True``, all ``put()`` calls are silent no-ops. - ``get()`` still works normally. Defaults to ``False``. - min_cache_size_bytes: When set to a positive integer, files whose - ``key.size`` is strictly below this threshold are not inserted. - ``None`` and ``0`` disable the threshold (default behaviour). - Negative values raise ``ValueError``. Defaults to ``None``. - """ - - def __init__( - self, - conninfo: str, - *, - read_only: bool = False, - min_cache_size_bytes: int | None = None, - ) -> None: - try: - import psycopg # noqa: F401 - except ImportError: - raise ImportError( - "PostgresHashCacher requires psycopg. " - "Install it with: pip install 'orcapod[postgresql]'" - ) from None - if min_cache_size_bytes is not None and min_cache_size_bytes < 0: - raise ValueError( - f"min_cache_size_bytes must be None or a non-negative integer, " - f"got {min_cache_size_bytes!r}" - ) - self._conninfo = conninfo - self._read_only = read_only - self._min_cache_size_bytes = min_cache_size_bytes - self._local = threading.local() - self._ensure_schema() - - def _ensure_schema(self) -> None: - """Create the cache table if it does not exist. - - Uses a dedicated one-shot connection so schema setup happens once - on construction, independent of the thread-local connection pool. - """ - import psycopg - - with psycopg.connect(self._conninfo) as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS file_hash_cache ( - path TEXT NOT NULL, - mtime_ns BIGINT NOT NULL, - size BIGINT NOT NULL, - hash BYTEA NOT NULL, - cached_at BIGINT NOT NULL - DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT, - PRIMARY KEY (path, mtime_ns, size) - ) - """ - ) - - def _connection(self) -> "_psycopg.Connection[tuple[object, ...]]": - """Return this thread's connection, opening it on first use.""" - import psycopg - - conn = getattr(self._local, "conn", None) - if conn is None: - conn = psycopg.connect(self._conninfo) - self._local.conn = conn - return conn - - def get(self, key: FileHashKey) -> ContentHash | None: - """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. - - Args: - key: File hash cache key. - - Returns: - Cached ``ContentHash``, or ``None`` if not found. - """ - conn = self._connection() - row = conn.execute( - "SELECT hash FROM file_hash_cache " - "WHERE path=%s AND mtime_ns=%s AND size=%s", - (str(key.path), key.mtime_ns, key.size), - ).fetchone() - if row is None: - return None - blob: bytes = bytes(row[0]) - method_bytes, digest = blob.split(b":", 1) - return ContentHash(method=method_bytes.decode("ascii"), digest=digest) - - def put(self, key: FileHashKey, value: ContentHash) -> None: - """Store ``value`` under ``key``. - - No-ops silently when ``read_only=True`` or when ``key.size`` is below - ``min_cache_size_bytes``. Uses ``INSERT ... ON CONFLICT DO NOTHING`` - so concurrent inserts of the same key from multiple workers are safe. - - Args: - key: File hash cache key. - value: ``ContentHash`` to store. - """ - if self._read_only: - return - if ( - self._min_cache_size_bytes is not None - and self._min_cache_size_bytes > 0 - and key.size < self._min_cache_size_bytes - ): - return - conn = self._connection() - conn.execute( - """ - INSERT INTO file_hash_cache (path, mtime_ns, size, hash) - VALUES (%s, %s, %s, %s) - ON CONFLICT (path, mtime_ns, size) DO NOTHING - """, - (str(key.path), key.mtime_ns, key.size, value.to_prefixed_digest()), - ) - conn.commit() - - def clear(self) -> None: - """Delete all rows from the cache table.""" - conn = self._connection() - conn.execute("DELETE FROM file_hash_cache") - conn.commit() - - def close(self) -> None: - """Close this thread's database connection.""" - conn = getattr(self._local, "conn", None) - if conn is not None: - conn.close() - self._local.conn = None - - def __enter__(self) -> "PostgresHashCacher": - """Return self for use as a context manager.""" - return self - - def __exit__(self, *_: object) -> None: - """Close the thread-local connection on exit.""" - self.close() - - def __repr__(self) -> str: - return ( - f"PostgresHashCacher(" - f"conninfo={self._conninfo!r}, " - f"read_only={self._read_only!r}, " - f"min_cache_size_bytes={self._min_cache_size_bytes!r})" - ) -``` - -- [ ] **Step 3.2: Run all Postgres tests** - -```bash -uv run pytest tests/test_hashing/test_postgres_hash_cacher.py -v -m postgres -``` - -Expected: all tests PASS. (Container spin-up takes ~10–20 s on first run.) - -- [ ] **Step 3.3: Commit** - -```bash -git add src/orcapod/hashing/postgres_hash_cacher.py tests/test_hashing/test_postgres_hash_cacher.py -git commit -m "feat(hashing): add PostgresHashCacher with thread-local psycopg3 connections (ITL-520)" -``` - ---- - -## Task 4: Export `PostgresHashCacher` from `orcapod.hashing` - -**Files:** -- Modify: `src/orcapod/hashing/__init__.py` - -- [ ] **Step 4.1: Add the guarded import** - -Open `src/orcapod/hashing/__init__.py`. Find the existing `try/except` block for `legacy_core` near the bottom of the import section (around line 86). Add a new `try/except` block immediately after the `from orcapod.hashing.hash_cachers import ...` line (around line 53): - -```python -from orcapod.hashing.hash_cachers import InMemoryHashCacher, SqliteHashCacher - -try: - from orcapod.hashing.postgres_hash_cacher import PostgresHashCacher -except ImportError: - PostgresHashCacher = None # type: ignore[assignment,misc] -``` - -Then add `"PostgresHashCacher"` to the `__all__` list at the bottom of the file, after `"SqliteHashCacher"`: - -```python - "InMemoryHashCacher", - "SqliteHashCacher", - "PostgresHashCacher", -``` - -- [ ] **Step 4.2: Verify the export works** - -```bash -uv run python -c "from orcapod.hashing import PostgresHashCacher; print(PostgresHashCacher)" -``` - -Expected: `` - -- [ ] **Step 4.3: Verify existing tests still pass** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py -v -``` - -Expected: all PASS (no regressions). - -- [ ] **Step 4.4: Commit** - -```bash -git add src/orcapod/hashing/__init__.py -git commit -m "feat(hashing): export PostgresHashCacher from orcapod.hashing (ITL-520)" -``` - ---- - -## Task 5: Extend `enable_file_hash_caching()` with `conninfo` parameter - -**Files:** -- Modify: `src/orcapod/contexts/__init__.py` -- Modify: `tests/test_hashing/test_hash_cachers.py` - -- [ ] **Step 5.1: Write the failing ValueError test first** - -Open `tests/test_hashing/test_hash_cachers.py`. Add this new class at the end of the file (after `TestEnableFileHashCaching`): - -```python -class TestEnableFileHashCachingConninfo: - def test_conninfo_and_db_path_raises(self, restore_default_file_handler, tmp_path): - """Providing both conninfo and db_path raises ValueError.""" - from orcapod.contexts import enable_file_hash_caching - - with pytest.raises(ValueError, match="not both"): - enable_file_hash_caching( - db_path=tmp_path / "x.db", - conninfo="postgresql://unused", - ) -``` - -- [ ] **Step 5.2: Run the failing test** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCachingConninfo -v -``` - -Expected: FAIL — `enable_file_hash_caching` does not yet accept `conninfo`. - -- [ ] **Step 5.3: Update `enable_file_hash_caching()`** - -Open `src/orcapod/contexts/__init__.py`. Find the `enable_file_hash_caching` function (around line 231). Replace the entire function signature and docstring with: - -```python -def enable_file_hash_caching( - *, - db_path: "Path | None" = None, - conninfo: str | None = None, - read_only: bool = False, - min_cache_size_bytes: int | None = None, -) -> None: - """Enable file hash caching on the default Orcapod context. - - When ``conninfo`` is provided, uses ``PostgresHashCacher`` for a shared, - network-accessible cache suitable for multi-machine pipelines. Otherwise - uses ``SqliteHashCacher`` for a local, per-machine cache (default). - - Call once at application startup before any file hashing occurs. - - If the handler already wraps a ``CachedFileHasher`` (i.e. this function - was already called), a warning is logged, all existing caching layers are - unwrapped to reach the original base hasher, and the new cacher is applied - around that base hasher. This keeps the system in a well-defined state - (exactly one caching layer) regardless of how many times this function - is called. - - For intentional multi-layer caching (e.g. in-memory L1 + SQLite L2), - construct a ``CachedFileHasher`` manually and register it directly via - the context's ``type_handler_registry`` instead. - - Also patches ``DirectoryHandler`` for ``orcapod.Directory``, using the - **same** ``CachedFileHasher`` instance. This means a file that was - cached via a direct ``op.File`` hash is also a cache hit when the same - file is encountered during directory traversal. - - Args: - db_path: Path to the SQLite cache database. Ignored when ``conninfo`` - is provided. Defaults to ``~/.orcapod/file_hash_cache.db`` or - the ``ORCAPOD_HASH_CACHE_DB`` environment variable. - conninfo: psycopg3 connection string for a PostgreSQL cache database, - e.g. ``"postgresql://user:pass@host:5432/db"``. When provided, - ``PostgresHashCacher`` is used instead of ``SqliteHashCacher``. - Mutually exclusive with ``db_path``. - read_only: When ``True``, the underlying cacher will not insert new - entries. Lookups still work normally. Defaults to ``False``. - min_cache_size_bytes: When set, files smaller than this byte count - are not inserted into the cache. ``None`` and ``0`` disable the - threshold. Defaults to ``None``. - - Raises: - ValueError: If both ``conninfo`` and ``db_path`` are provided. - """ - if conninfo is not None and db_path is not None: - raise ValueError( - "enable_file_hash_caching(): provide conninfo or db_path, not both." - ) -``` - -Then, inside the function body, replace the block that constructs `cached_file_hasher` (find the lines that build `SqliteHashCacher` and `CachedFileHasher`): - -```python - if conninfo is not None: - from orcapod.hashing.postgres_hash_cacher import PostgresHashCacher - - cacher = PostgresHashCacher( - conninfo, - read_only=read_only, - min_cache_size_bytes=min_cache_size_bytes, - ) - else: - # SqliteHashCacher import was previously at the top of the function body; - # it now lives inside this branch only. - from orcapod.hashing.hash_cachers import SqliteHashCacher - - cacher = SqliteHashCacher( - db_path, - read_only=read_only, - min_cache_size_bytes=min_cache_size_bytes, - ) - - cached_file_hasher = CachedFileHasher( - file_hasher=base_hasher, - cacher=cacher, - ) -``` - -The rest of the function body (registering `FileHandler` and `DirectoryHandler`) remains unchanged. - -- [ ] **Step 5.4: Run the ValueError test — should now pass** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCachingConninfo -v -``` - -Expected: PASS - -- [ ] **Step 5.5: Verify no regressions in existing enable_file_hash_caching tests** - -```bash -uv run pytest tests/test_hashing/test_hash_cachers.py -v -``` - -Expected: all PASS - -- [ ] **Step 5.6: Add Postgres-specific enable_file_hash_caching tests to the Postgres test file** - -Open `tests/test_hashing/test_postgres_hash_cacher.py`. Append this class at the end: - -```python -# --------------------------------------------------------------------------- -# enable_file_hash_caching(conninfo=...) integration -# --------------------------------------------------------------------------- - - -@pytest.fixture() -def restore_default_file_handler(): - """Restore the default FileHandler and DirectoryHandler after each test.""" - from orcapod.contexts import get_default_context - from orcapod.extension_types.directory_type import Directory - from orcapod.extension_types.file_type import File - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - original_file_handler = registry.get_handler_for_type(File) - original_dir_handler = registry.get_handler_for_type(Directory) - yield - registry.register(File, original_file_handler) - registry.register(Directory, original_dir_handler) - - -@pytest.mark.postgres -class TestEnableFileHashCachingPostgres: - def test_conninfo_activates_postgres_cacher( - self, restore_default_file_handler, pg_conninfo - ): - """enable_file_hash_caching(conninfo=...) wires up PostgresHashCacher.""" - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - - enable_file_hash_caching(conninfo=pg_conninfo) - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - assert isinstance(handler.file_hasher.cacher, PostgresHashCacher) - - def test_conninfo_passes_read_only( - self, restore_default_file_handler, pg_conninfo - ): - """read_only=True is forwarded to PostgresHashCacher.""" - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - - enable_file_hash_caching(conninfo=pg_conninfo, read_only=True) - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - assert handler.file_hasher.cacher._read_only is True - - def test_conninfo_passes_min_cache_size_bytes( - self, restore_default_file_handler, pg_conninfo - ): - """min_cache_size_bytes is forwarded to PostgresHashCacher.""" - from orcapod.contexts import enable_file_hash_caching, get_default_context - from orcapod.extension_types.file_type import File - - enable_file_hash_caching(conninfo=pg_conninfo, min_cache_size_bytes=2048) - - context = get_default_context() - registry = context.semantic_hasher.type_handler_registry - handler = registry.get_handler_for_type(File) - assert handler.file_hasher.cacher._min_cache_size_bytes == 2048 -``` - -- [ ] **Step 5.7: Run the new integration tests** - -```bash -uv run pytest tests/test_hashing/test_postgres_hash_cacher.py::TestEnableFileHashCachingPostgres -v -m postgres -``` - -Expected: all PASS - -- [ ] **Step 5.8: Commit** - -```bash -git add src/orcapod/contexts/__init__.py \ - tests/test_hashing/test_hash_cachers.py \ - tests/test_hashing/test_postgres_hash_cacher.py -git commit -m "feat(contexts): add conninfo param to enable_file_hash_caching() for Postgres backend (ITL-520)" -``` - ---- - -## Task 6: Update documentation - -**Files:** -- Modify: `docs/concepts/file-hash-caching.md` - -- [ ] **Step 6.1: Add the backend comparison section** - -Open `docs/concepts/file-hash-caching.md`. Find the end of the file and append the following section. (If there is already a section about "Controlling when the cache is written" from ITL-519, add the new section after it.) - -```markdown -## Choosing a backend: SQLite vs Postgres - -Orcapod ships two hash cache backends. Pick based on your deployment: - -| | SQLite | Postgres | -|---|---|---| -| Setup | Zero — file on disk | Requires a running Postgres server | -| Scope | Per-machine | Shared across machines and pipeline runs | -| Concurrency | Single-writer | Multi-writer safe (`ON CONFLICT DO NOTHING`) | -| Best for | Local development, single-machine pipelines | Distributed pipelines, shared caches | - -### Using the SQLite backend (default) - -```python -import orcapod as op - -op.enable_file_hash_caching() # uses ~/.orcapod/file_hash_cache.db -# or specify a path: -op.enable_file_hash_caching(db_path="/shared/nfs/cache.db") -``` - -### Using the Postgres backend - -Requires `psycopg` (install with `pip install 'orcapod[postgresql]'`): - -```python -import orcapod as op - -op.enable_file_hash_caching( - conninfo="postgresql://user:pass@db-host:5432/orcapod_cache" -) -``` - -You can also construct `PostgresHashCacher` directly and pass it to -`CachedFileHasher` for advanced configurations: - -```python -from orcapod.hashing import CachedFileHasher, FileHasher, PostgresHashCacher - -cacher = PostgresHashCacher( - conninfo="postgresql://user:pass@db-host:5432/orcapod_cache", - read_only=True, # lookup only, no writes - min_cache_size_bytes=1_048_576, # skip files < 1 MB -) -hasher = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) -``` - -### Schema - -`PostgresHashCacher` creates the following table on first use -(`CREATE TABLE IF NOT EXISTS`): - -```sql -CREATE TABLE IF NOT EXISTS file_hash_cache ( - path TEXT NOT NULL, - mtime_ns BIGINT NOT NULL, - size BIGINT NOT NULL, - hash BYTEA NOT NULL, - cached_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT, - PRIMARY KEY (path, mtime_ns, size) -) -``` - -Minimum supported Postgres version: **14**. -``` - -- [ ] **Step 6.2: Commit** - -```bash -git add docs/concepts/file-hash-caching.md -git commit -m "docs(hashing): add Postgres backend section to file-hash-caching docs (ITL-520)" -``` - ---- - -## Task 7: Full test run and PR - -**Files:** none - -- [ ] **Step 7.1: Run the full hashing test suite** - -```bash -uv run pytest tests/test_hashing/ -v -``` - -Expected: all tests PASS (Postgres tests require Docker; SQLite/in-memory tests run unconditionally). - -- [ ] **Step 7.2: Run the broader test suite for regressions** - -```bash -uv run pytest tests/ -v --ignore=tests/test_hashing/test_postgres_hash_cacher.py -q -``` - -Expected: all tests PASS, no regressions. - -- [ ] **Step 7.3: Push and open PR** - -```bash -git push -u origin eywalker/itl-520-filehasher-add-postgresql-backend-for-the-hash-cacher -``` - -Then open a PR targeting `main` with a description referencing `Closes ITL-520`. diff --git a/superpowers/plans/2026-07-14-itl-523-post-run-hook.md b/superpowers/plans/2026-07-14-itl-523-post-run-hook.md deleted file mode 100644 index 3a0ba5be0..000000000 --- a/superpowers/plans/2026-07-14-itl-523-post-run-hook.md +++ /dev/null @@ -1,1374 +0,0 @@ -# Post-Run Hook for Function Pods — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a per-pod post-run hook mechanism to `FunctionPod` so registered callables fire after every invocation with a typed payload describing what ran. - -**Architecture:** New types live in `src/orcapod/hooks.py`. `_FunctionPodBase` gains a mutable hook list plus an `_invoke_with_hooks()` wrapper method; all call sites that previously called `process_data()` directly now call `_invoke_with_hooks()` instead. `CachedFunctionPod` overrides `_invoke_with_hooks()` to supply the correct `InvocationStatus.HIT` when a database cache hit occurs. - -**Tech Stack:** Python 3.11+, PyArrow, pytest, uv (all commands via `uv run`) - -**Spec:** `superpowers/specs/2026-07-14-itl-523-post-run-hook-design.md` - ---- - -## File Map - -| Action | Path | Responsibility | -|--------|------|----------------| -| **Create** | `src/orcapod/hooks.py` | All public hook types: `InvocationStatus`, `RunStats`, `PodContext`, `PostRunPayload`, `PostRunHookFn`, `HookConfig`, `PostRunHook` | -| **Modify** | `src/orcapod/core/function_pod.py` | Add `_post_run_hooks`, `add_post_run_hook()`, `_fire_post_run_hooks()`, `_invoke_with_hooks()`, `_async_invoke_with_hooks()` to `_FunctionPodBase`; update call sites in `FunctionPodStream._iter_data_sequential`, `_iter_data_concurrent`, and `async_execute` | -| **Modify** | `src/orcapod/core/cached_function_pod.py` | Override `_invoke_with_hooks()` and `_async_invoke_with_hooks()` to detect `InvocationStatus.HIT` via `RESULT_COMPUTED_FLAG` | -| **Modify** | `src/orcapod/__init__.py` | Re-export all public hook types | -| **Create** | `tests/test_core/function_pod/test_post_run_hooks.py` | Full test suite (10 test classes) | - ---- - -## Task 1: Create `src/orcapod/hooks.py` - -**Files:** -- Create: `src/orcapod/hooks.py` - -- [ ] **Step 1: Create the hooks module** - -```python -# src/orcapod/hooks.py -"""Post-run hook types for function pods. - -Defines the payload, status, and hook configuration types used by the -post-run hook mechanism on function pods. Import these when writing or -registering hooks. -""" - -from __future__ import annotations - -import dataclasses -from collections.abc import Callable -from datetime import datetime -from enum import Enum -from typing import TYPE_CHECKING, Literal - -if TYPE_CHECKING: - from orcapod.protocols.core_protocols import DataProtocol, TagProtocol - - -class InvocationStatus(str, Enum): - """Status of a single function pod invocation. - - Attributes: - COMPUTED: The function was invoked and produced a fresh result. - HIT: The result was served from a pod-level database cache - (``CachedFunctionPod``). - ERROR: The function raised an exception. - """ - - COMPUTED = "computed" - HIT = "hit" - ERROR = "error" - - -@dataclasses.dataclass(frozen=True) -class RunStats: - """Timing and status information for a single pod invocation. - - Attributes: - duration_ms: Wall-clock time in milliseconds. - status: Whether the result was freshly computed, a cache hit, or an error. - started_at: UTC timestamp when the invocation started. - finished_at: UTC timestamp when the invocation finished (after hooks fire). - error: The exception raised, if ``status == ERROR``; ``None`` otherwise. - """ - - duration_ms: float - status: InvocationStatus - started_at: datetime - finished_at: datetime - error: Exception | None = None - - -@dataclasses.dataclass(frozen=True) -class PodContext: - """Identity information about the pod that produced a result. - - Attributes: - label: Human-readable pod label (``pod.label``); ``None`` if not set. - pod_hash: Hex-string content hash of the pod (``pod.content_hash().to_string()``). - Changes when the underlying function code or version changes. - """ - - label: str | None - pod_hash: str - - -@dataclasses.dataclass(frozen=True) -class PostRunPayload: - """Payload passed to every post-run hook after a pod invocation. - - Attributes: - record_id_hash: String form of ``output.datagram_uuid`` — the same UUID - used as the primary key when the result is stored in a backing database. - ``None`` when ``output`` is ``None`` (filtered row or error). - tag: The input tag for this invocation. Treat as read-only. - input: The input data for this invocation. Treat as read-only. - output: The output data; ``None`` if the function filtered the row out or raised. - stats: Timing and status bundle. - pod: Identity of the pod that produced this result. - """ - - record_id_hash: str | None - tag: TagProtocol - input: DataProtocol - output: DataProtocol | None - stats: RunStats - pod: PodContext - - -PostRunHookFn = Callable[["PostRunPayload"], None] -"""A plain hook callable: ``(PostRunPayload) -> None``. - -Defaults to fail-loud on error (exceptions propagate). -""" - - -@dataclasses.dataclass(frozen=True) -class HookConfig: - """Hook callable with explicit error-handling behaviour. - - Use this instead of a plain callable when you want the hook to log and - continue on failure rather than propagating the exception. - - Attributes: - fn: The hook callable. - on_error: ``"raise"`` (default) propagates exceptions; ``"log"`` logs at - WARNING level and continues. - - Example: - pod.add_post_run_hook(HookConfig(fn=my_hook, on_error="log")) - """ - - fn: PostRunHookFn - on_error: Literal["raise", "log"] = "raise" - - -PostRunHook = PostRunHookFn | HookConfig -"""A hook is either a plain callable (fail-loud) or a ``HookConfig`` wrapper.""" -``` - -- [ ] **Step 2: Verify module imports cleanly** - -```bash -uv run python -c "from orcapod.hooks import PostRunPayload, HookConfig, InvocationStatus, RunStats, PodContext, PostRunHook, PostRunHookFn; print('OK')" -``` - -Expected: `OK` - -- [ ] **Step 3: Commit** - -```bash -cd /home/kurouto/kurouto-jobs/09c4ea39-6353-4e02-ba3a-a9b86b0e157b/orcapod-python -git add src/orcapod/hooks.py -git commit -m "feat(hooks): add PostRunPayload and hook types module (ITL-523)" -``` - ---- - -## Task 2: Hook infrastructure on `_FunctionPodBase` + sequential call site - -Add `_post_run_hooks`, `add_post_run_hook`, `_fire_post_run_hooks`, and `_invoke_with_hooks` to `_FunctionPodBase`. Update `FunctionPodStream._iter_data_sequential` to call `_invoke_with_hooks`. This covers tests 1–4, 6, and 7. - -**Files:** -- Create: `tests/test_core/function_pod/test_post_run_hooks.py` -- Modify: `src/orcapod/core/function_pod.py` - -- [ ] **Step 1: Create the test file with failing tests** - -```python -# tests/test_core/function_pod/test_post_run_hooks.py -"""Tests for FunctionPod post-run hooks (ITL-523). - -Covers: registration, firing order, failure semantics, payload correctness, -filtered output, error status, cache hit status, parallel execution, -decorator convenience, and empty-hooks no-overhead path. -""" - -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.cached_function_pod import CachedFunctionPod -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod, function_pod -from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.databases import InMemoryArrowDatabase -from orcapod.hooks import HookConfig, InvocationStatus, PostRunPayload - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_stream(n: int = 2) -> ArrowTableStream: - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("x", pa.int64(), nullable=False), - ]) - table = pa.table( - { - "id": pa.array(list(range(n)), type=pa.int64()), - "x": pa.array(list(range(n)), type=pa.int64()), - }, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -def double(x: int) -> int: - return x * 2 - - -def _make_double_pod() -> FunctionPod: - pf = PythonDataFunction(double, output_keys="result") - return FunctionPod(pf) - - -# --------------------------------------------------------------------------- -# 1. Single hook fires with correct payload -# --------------------------------------------------------------------------- - - -class TestSingleHookPayload: - def test_hook_fires_with_correct_payload(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - assert len(payloads) == 1 - p = payloads[0] - assert p.stats.status == InvocationStatus.COMPUTED - assert p.stats.duration_ms >= 0 - assert p.stats.error is None - assert p.output is not None - assert p.record_id_hash == str(p.output.datagram_uuid) - assert p.pod.label == pod.label - assert p.pod.pod_hash == pod.content_hash().to_string() - assert p.input is not None - assert p.tag is not None - - def test_hook_fires_for_each_row(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=3)) - list(stream.iter_data()) - - assert len(payloads) == 3 - - -# --------------------------------------------------------------------------- -# 2. Multiple hooks fire in registration order -# --------------------------------------------------------------------------- - - -class TestHookOrdering: - def test_multiple_hooks_fire_in_order(self): - pod = _make_double_pod() - fired: list[str] = [] - pod.add_post_run_hook(lambda p: fired.append("first")) - pod.add_post_run_hook(lambda p: fired.append("second")) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - assert fired == ["first", "second"] - - def test_hooks_fire_in_order_for_every_row(self): - pod = _make_double_pod() - fired: list[str] = [] - pod.add_post_run_hook(lambda p: fired.append("first")) - pod.add_post_run_hook(lambda p: fired.append("second")) - - stream = pod.process(_make_stream(n=2)) - list(stream.iter_data()) - - assert fired == ["first", "second", "first", "second"] - - -# --------------------------------------------------------------------------- -# 3. Fail-loud hook error -# --------------------------------------------------------------------------- - - -class TestHookFailureFast: - def test_failing_hook_propagates_exception(self): - pod = _make_double_pod() - - def bad_hook(p: PostRunPayload) -> None: - raise ValueError("hook exploded") - - pod.add_post_run_hook(bad_hook) - - stream = pod.process(_make_stream(n=1)) - with pytest.raises(ValueError, match="hook exploded"): - list(stream.iter_data()) - - def test_failing_hook_stops_remaining_hooks(self): - pod = _make_double_pod() - second_fired: list[bool] = [] - - def bad_hook(p: PostRunPayload) -> None: - raise ValueError("stops here") - - pod.add_post_run_hook(bad_hook) - pod.add_post_run_hook(lambda p: second_fired.append(True)) - - stream = pod.process(_make_stream(n=1)) - with pytest.raises(ValueError): - list(stream.iter_data()) - - assert second_fired == [] - - -# --------------------------------------------------------------------------- -# 4. Resilient hook error -# --------------------------------------------------------------------------- - - -class TestHookFailureResilient: - def test_resilient_hook_suppresses_exception(self): - pod = _make_double_pod() - second_fired: list[bool] = [] - - def bad_hook(p: PostRunPayload) -> None: - raise ValueError("suppressed") - - def second_hook(p: PostRunPayload) -> None: - second_fired.append(True) - - pod.add_post_run_hook(HookConfig(fn=bad_hook, on_error="log")) - pod.add_post_run_hook(second_hook) - - stream = pod.process(_make_stream(n=1)) - results = list(stream.iter_data()) - - assert results # computation result returned - assert second_fired == [True] # next hook still fired - - def test_hookconfig_raise_is_same_as_plain_callable(self): - pod = _make_double_pod() - - def bad_hook(p: PostRunPayload) -> None: - raise ValueError("still loud") - - pod.add_post_run_hook(HookConfig(fn=bad_hook, on_error="raise")) - - stream = pod.process(_make_stream(n=1)) - with pytest.raises(ValueError, match="still loud"): - list(stream.iter_data()) - - -# --------------------------------------------------------------------------- -# 6. Error status (pod function raises) -# --------------------------------------------------------------------------- - - -class TestErrorStatus: - def test_pod_error_fires_hook_with_error_status(self): - def explodes(x: int) -> int: - raise RuntimeError("boom") - - pf = PythonDataFunction(explodes, output_keys="result") - pod = FunctionPod(pf) - - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(HookConfig(fn=payloads.append, on_error="log")) - - stream = pod.process(_make_stream(n=1)) - with pytest.raises(RuntimeError, match="boom"): - list(stream.iter_data()) - - assert len(payloads) == 1 - p = payloads[0] - assert p.stats.status == InvocationStatus.ERROR - assert isinstance(p.stats.error, RuntimeError) - assert str(p.stats.error) == "boom" - assert p.output is None - assert p.record_id_hash is None - - def test_original_exception_reraises_after_hooks(self): - def explodes(x: int) -> int: - raise RuntimeError("original") - - pf = PythonDataFunction(explodes, output_keys="result") - pod = FunctionPod(pf) - pod.add_post_run_hook(HookConfig(fn=lambda p: None, on_error="log")) - - stream = pod.process(_make_stream(n=1)) - with pytest.raises(RuntimeError, match="original"): - list(stream.iter_data()) - - -# --------------------------------------------------------------------------- -# 7. Filtered output -# --------------------------------------------------------------------------- - - -class TestFilteredOutput: - def test_filtered_row_fires_hook_with_none_output(self): - def filter_all(x: int) -> int | None: - return None - - pf = PythonDataFunction(filter_all, output_keys="result") - pod = FunctionPod(pf) - - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - results = list(stream.iter_data()) - - assert results == [] - assert len(payloads) == 1 - p = payloads[0] - assert p.output is None - assert p.record_id_hash is None - assert p.stats.status == InvocationStatus.COMPUTED - assert p.stats.error is None - - -# --------------------------------------------------------------------------- -# 10. Empty hooks — no overhead path -# --------------------------------------------------------------------------- - - -class TestEmptyHooks: - def test_pod_with_no_hooks_has_empty_list(self): - pod = _make_double_pod() - assert pod._post_run_hooks == [] - - def test_pod_with_no_hooks_processes_normally(self): - pod = _make_double_pod() - stream = pod.process(_make_stream(n=2)) - results = list(stream.iter_data()) - assert len(results) == 2 -``` - -- [ ] **Step 2: Run the tests to confirm they fail with AttributeError** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py -x -q 2>&1 | head -20 -``` - -Expected: `AttributeError: '_FunctionPodBase' object has no attribute 'add_post_run_hook'` (or similar) - -- [ ] **Step 3: Add imports to `function_pod.py`** - -At the top of `src/orcapod/core/function_pod.py`, add these imports after the existing import block (after line ~39): - -```python -from datetime import datetime, timezone - -from orcapod.hooks import ( - HookConfig, - InvocationStatus, - PodContext, - PostRunPayload, - PostRunHook, - RunStats, -) -``` - -- [ ] **Step 4: Add `_post_run_hooks` initialisation to `_FunctionPodBase.__init__`** - -In `_FunctionPodBase.__init__` (around line 76, after `self._data_function = data_function`), add: - -```python - self._post_run_hooks: list[PostRunHook] = [] -``` - -- [ ] **Step 5: Add `add_post_run_hook`, `_fire_post_run_hooks`, and `_invoke_with_hooks` to `_FunctionPodBase`** - -After the `process_data` method (around line 173 in `function_pod.py`), insert these three methods: - -```python - def add_post_run_hook(self, hook: PostRunHook) -> None: - """Register a post-run hook on this pod. - - Hooks fire after every invocation (computed, cache hit, or error), in - registration order, before the result is emitted downstream. - - A plain callable defaults to fail-loud (exceptions propagate, stopping - the pod run). Wrap in ``HookConfig(fn=..., on_error="log")`` to log and - continue on hook failure. - - Args: - hook: A callable ``(PostRunPayload) -> None``, or a ``HookConfig`` - wrapping such a callable with explicit error handling. - """ - self._post_run_hooks.append(hook) - - def _fire_post_run_hooks(self, payload: PostRunPayload) -> None: - """Fire all registered hooks with payload in registration order. - - Args: - payload: The post-run payload to pass to each hook. - """ - for hook in self._post_run_hooks: - fn = hook.fn if isinstance(hook, HookConfig) else hook - on_error = hook.on_error if isinstance(hook, HookConfig) else "raise" - try: - fn(payload) - except Exception as exc: - if on_error == "raise": - raise - logger.warning( - "Post-run hook %r raised and was suppressed: %s", - fn, - exc, - exc_info=True, - ) - - def _invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - """Call ``process_data``, time it, and fire post-run hooks. - - This is the call site used by ``FunctionPodStream`` and - ``async_execute``; ``process_data`` itself is unchanged. Override in - subclasses (e.g. ``CachedFunctionPod``) to supply a different - ``InvocationStatus``. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional data execution logger forwarded to ``process_data``. - - Returns: - A ``(tag, output_data)`` tuple. - """ - started_at = datetime.now(timezone.utc) - exc: Exception | None = None - out_tag = tag - output_data: DataProtocol | None = None - - try: - out_tag, output_data = self.process_data(tag, data, logger=logger) - status = InvocationStatus.COMPUTED - except Exception as e: - exc = e - status = InvocationStatus.ERROR - - finished_at = datetime.now(timezone.utc) - - if self._post_run_hooks: - record_id = ( - str(output_data.datagram_uuid) if output_data is not None else None - ) - payload = PostRunPayload( - record_id_hash=record_id, - tag=tag, - input=data, - output=output_data, - stats=RunStats( - duration_ms=(finished_at - started_at).total_seconds() * 1000, - status=status, - started_at=started_at, - finished_at=finished_at, - error=exc, - ), - pod=PodContext( - label=self.label, - pod_hash=self.content_hash().to_string(), - ), - ) - self._fire_post_run_hooks(payload) - - if exc is not None: - raise exc - return out_tag, output_data - - async def _async_invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - """Async counterpart of ``_invoke_with_hooks``. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional data execution logger forwarded to ``async_process_data``. - - Returns: - A ``(tag, output_data)`` tuple. - """ - started_at = datetime.now(timezone.utc) - exc: Exception | None = None - out_tag = tag - output_data: DataProtocol | None = None - - try: - out_tag, output_data = await self.async_process_data( - tag, data, logger=logger - ) - status = InvocationStatus.COMPUTED - except Exception as e: - exc = e - status = InvocationStatus.ERROR - - finished_at = datetime.now(timezone.utc) - - if self._post_run_hooks: - record_id = ( - str(output_data.datagram_uuid) if output_data is not None else None - ) - payload = PostRunPayload( - record_id_hash=record_id, - tag=tag, - input=data, - output=output_data, - stats=RunStats( - duration_ms=(finished_at - started_at).total_seconds() * 1000, - status=status, - started_at=started_at, - finished_at=finished_at, - error=exc, - ), - pod=PodContext( - label=self.label, - pod_hash=self.content_hash().to_string(), - ), - ) - self._fire_post_run_hooks(payload) - - if exc is not None: - raise exc - return out_tag, output_data -``` - -- [ ] **Step 6: Update `FunctionPodStream._iter_data_sequential` to call `_invoke_with_hooks`** - -In `FunctionPodStream._iter_data_sequential` (around line 530), replace: - -```python - tag, output_data = self._function_pod.process_data(tag, data) -``` - -with: - -```python - tag, output_data = self._function_pod._invoke_with_hooks(tag, data) -``` - -- [ ] **Step 7: Run the tests — most should pass now** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py -x -q 2>&1 | head -30 -``` - -Expected: `TestSingleHookPayload`, `TestHookOrdering`, `TestHookFailureFast`, `TestHookFailureResilient`, `TestErrorStatus`, `TestFilteredOutput`, `TestEmptyHooks` all pass. `TestCacheHitStatus`, `TestParallelExecution`, `TestDecoratorConvenience` will be added in later tasks. - -- [ ] **Step 8: Run the existing function pod test suite to confirm no regressions** - -```bash -uv run pytest tests/test_core/function_pod/ -q 2>&1 | tail -10 -``` - -Expected: all existing tests pass. - -- [ ] **Step 9: Commit** - -```bash -git add src/orcapod/core/function_pod.py tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "feat(function_pod): add post-run hook infrastructure and sequential call site (ITL-523)" -``` - ---- - -## Task 3: Update `FunctionPodStream._iter_data_concurrent` + parallel test - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` (two lines in `_iter_data_concurrent`) -- Modify: `tests/test_core/function_pod/test_post_run_hooks.py` (add `TestParallelExecution`) - -- [ ] **Step 1: Add `TestParallelExecution` to the test file** - -Append to `tests/test_core/function_pod/test_post_run_hooks.py`: - -```python -# --------------------------------------------------------------------------- -# 8. Parallel execution (concurrent path via _iter_data_concurrent) -# --------------------------------------------------------------------------- - - -class _ConcurrentExecutor: - """Minimal executor that marks supports_concurrent_execution=True.""" - - @property - def executor_type_id(self) -> str: - return "test-concurrent" - - def supported_function_type_ids(self) -> frozenset[str]: - return frozenset() - - @property - def supports_concurrent_execution(self) -> bool: - return True - - def execute(self, data_function, data): - return data_function.direct_call(data) - - async def async_execute(self, data_function, data): - return data_function.direct_call(data) - - def execute_callable(self, fn, kwargs, executor_options=None, **kw): - return fn(**kwargs) - - async def async_execute_callable(self, fn, kwargs, executor_options=None, **kw): - return fn(**kwargs) - - def with_options(self, **kwargs): - return self - - -class TestParallelExecution: - def test_hooks_fire_for_all_inputs_under_concurrent_executor(self): - executor = _ConcurrentExecutor() - pf = PythonDataFunction(double, output_keys="result", executor=executor) - pod = FunctionPod(pf) - - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=4)) - results = list(stream.iter_data()) - - assert len(results) == 4 - assert len(payloads) == 4 - assert all( - p.stats.status == InvocationStatus.COMPUTED for p in payloads - ) -``` - -- [ ] **Step 2: Run the new test to confirm it fails** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestParallelExecution -x -q 2>&1 | head -15 -``` - -Expected: FAIL — hooks fire 0 times (concurrent path still calls `process_data` / `async_process_data` directly). - -- [ ] **Step 3: Update `_iter_data_concurrent` — sync fallback path** - -In `FunctionPodStream._iter_data_concurrent` (around line 564), replace: - -```python - results = [ - self._function_pod.process_data(tag, pkt) - for _, tag, pkt in to_compute - ] -``` - -with: - -```python - results = [ - self._function_pod._invoke_with_hooks(tag, pkt) - for _, tag, pkt in to_compute - ] -``` - -- [ ] **Step 4: Update `_iter_data_concurrent` — async gather path** - -In `FunctionPodStream._iter_data_concurrent` (around line 570), replace: - -```python - async def _gather() -> list[tuple[TagProtocol, DataProtocol | None]]: - return list( - await asyncio.gather( - *[ - self._function_pod.async_process_data(tag, pkt) - for _, tag, pkt in to_compute - ] - ) - ) -``` - -with: - -```python - async def _gather() -> list[tuple[TagProtocol, DataProtocol | None]]: - return list( - await asyncio.gather( - *[ - self._function_pod._async_invoke_with_hooks(tag, pkt) - for _, tag, pkt in to_compute - ] - ) - ) -``` - -- [ ] **Step 5: Run the parallel test — should pass now** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestParallelExecution -x -q -``` - -Expected: PASS - -- [ ] **Step 6: Run full function pod suite for regressions** - -```bash -uv run pytest tests/test_core/function_pod/ -q 2>&1 | tail -10 -``` - -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/function_pod.py tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "feat(function_pod): update concurrent call sites to use _invoke_with_hooks (ITL-523)" -``` - ---- - -## Task 4: Update `async_execute` call site - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` (`async_execute`) -- Modify: `tests/test_core/function_pod/test_post_run_hooks.py` (add `TestAsyncExecuteHooks`) - -- [ ] **Step 1: Add `TestAsyncExecuteHooks` to the test file** - -Append to `tests/test_core/function_pod/test_post_run_hooks.py`: - -```python -# --------------------------------------------------------------------------- -# async_execute path -# --------------------------------------------------------------------------- - - -class TestAsyncExecuteHooks: - @pytest.mark.asyncio - async def test_hooks_fire_through_async_execute(self): - from orcapod.channels import Channel - - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = _make_stream(n=3) - input_ch: Channel = Channel() - output_ch: Channel = Channel() - - async def feed() -> None: - for tag, data in stream.iter_data(): - await input_ch.writer.send((tag, data)) - await input_ch.writer.close() - - import asyncio - await asyncio.gather( - feed(), - pod.async_execute([input_ch.reader], output_ch.writer), - ) - - results = [] - async for item in output_ch.reader: - results.append(item) - - assert len(results) == 3 - assert len(payloads) == 3 - assert all(p.stats.status == InvocationStatus.COMPUTED for p in payloads) -``` - -- [ ] **Step 2: Run to confirm it fails** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestAsyncExecuteHooks -x -q 2>&1 | head -15 -``` - -Expected: FAIL — hooks fire 0 times (async_execute still calls `async_process_data` directly). - -- [ ] **Step 3: Update `async_execute` to call `_async_invoke_with_hooks`** - -In `_FunctionPodBase.async_execute`, inside the `process_one` inner function (around line 241), replace: - -```python - out_tag, result_data = await self.async_process_data( - tag, data, logger=pkt_logger - ) -``` - -with: - -```python - out_tag, result_data = await self._async_invoke_with_hooks( - tag, data, logger=pkt_logger - ) -``` - -- [ ] **Step 4: Run the async_execute test — should pass** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestAsyncExecuteHooks -x -q -``` - -Expected: PASS - -- [ ] **Step 5: Run full function pod suite for regressions** - -```bash -uv run pytest tests/test_core/function_pod/ -q 2>&1 | tail -10 -``` - -Expected: all pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/function_pod.py tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "feat(function_pod): update async_execute to use _async_invoke_with_hooks (ITL-523)" -``` - ---- - -## Task 5: `CachedFunctionPod` override for cache hit status - -**Files:** -- Modify: `src/orcapod/core/cached_function_pod.py` -- Modify: `tests/test_core/function_pod/test_post_run_hooks.py` (add `TestCacheHitStatus`) - -- [ ] **Step 1: Add `TestCacheHitStatus` to the test file** - -Append to `tests/test_core/function_pod/test_post_run_hooks.py`: - -```python -# --------------------------------------------------------------------------- -# 5. Cache hit status -# --------------------------------------------------------------------------- - - -class TestCacheHitStatus: - def test_second_call_fires_hook_with_hit_status(self): - inner = _make_double_pod() - db = InMemoryArrowDatabase() - pod = CachedFunctionPod(function_pod=inner, result_database=db) - - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream1 = pod.process(_make_stream(n=1)) - list(stream1.iter_data()) - - stream2 = pod.process(_make_stream(n=1)) - list(stream2.iter_data()) - - assert len(payloads) == 2 - assert payloads[0].stats.status == InvocationStatus.COMPUTED - assert payloads[1].stats.status == InvocationStatus.HIT - - def test_cache_hit_payload_has_record_id(self): - inner = _make_double_pod() - db = InMemoryArrowDatabase() - pod = CachedFunctionPod(function_pod=inner, result_database=db) - - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream1 = pod.process(_make_stream(n=1)) - list(stream1.iter_data()) - stream2 = pod.process(_make_stream(n=1)) - list(stream2.iter_data()) - - assert payloads[1].record_id_hash is not None - assert payloads[1].output is not None -``` - -- [ ] **Step 2: Run to confirm it fails** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestCacheHitStatus -x -q 2>&1 | head -15 -``` - -Expected: FAIL — `payloads[1].stats.status == InvocationStatus.COMPUTED` instead of `HIT` (base class always returns `COMPUTED`). - -- [ ] **Step 3: Add imports to `cached_function_pod.py`** - -At the top of `src/orcapod/core/cached_function_pod.py`, add after existing imports: - -```python -from datetime import datetime, timezone - -from orcapod.hooks import ( - HookConfig, - InvocationStatus, - PodContext, - PostRunPayload, - RunStats, -) -from orcapod.types import ColumnConfig -``` - -- [ ] **Step 4: Add `_invoke_with_hooks` override to `CachedFunctionPod`** - -After the `async_process_data` method in `CachedFunctionPod` (around line 149), add: - -```python - def _invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - """Override to detect cache hit status from ``RESULT_COMPUTED_FLAG`` meta. - - Calls ``self.process_data()`` (which owns all cache lookup and store - logic), then reads ``RESULT_COMPUTED_FLAG`` from the output data meta to - determine ``InvocationStatus.HIT`` vs ``InvocationStatus.COMPUTED``. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional data execution logger. - - Returns: - A ``(tag, output_data)`` tuple. - """ - started_at = datetime.now(timezone.utc) - exc: Exception | None = None - out_tag = tag - output_data: DataProtocol | None = None - status = InvocationStatus.COMPUTED - - try: - out_tag, output_data = self.process_data(tag, data, logger=logger) - if output_data is not None: - meta = output_data.as_dict(columns=ColumnConfig(meta=True)) - if meta.get(self.RESULT_COMPUTED_FLAG) is False: - status = InvocationStatus.HIT - except Exception as e: - exc = e - status = InvocationStatus.ERROR - - finished_at = datetime.now(timezone.utc) - - if self._post_run_hooks: - record_id = ( - str(output_data.datagram_uuid) if output_data is not None else None - ) - payload = PostRunPayload( - record_id_hash=record_id, - tag=tag, - input=data, - output=output_data, - stats=RunStats( - duration_ms=(finished_at - started_at).total_seconds() * 1000, - status=status, - started_at=started_at, - finished_at=finished_at, - error=exc, - ), - pod=PodContext( - label=self.label, - pod_hash=self.content_hash().to_string(), - ), - ) - self._fire_post_run_hooks(payload) - - if exc is not None: - raise exc - return out_tag, output_data - - async def _async_invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - """Async counterpart of ``_invoke_with_hooks`` for ``CachedFunctionPod``. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional data execution logger. - - Returns: - A ``(tag, output_data)`` tuple. - """ - started_at = datetime.now(timezone.utc) - exc: Exception | None = None - out_tag = tag - output_data: DataProtocol | None = None - status = InvocationStatus.COMPUTED - - try: - out_tag, output_data = await self.async_process_data( - tag, data, logger=logger - ) - if output_data is not None: - meta = output_data.as_dict(columns=ColumnConfig(meta=True)) - if meta.get(self.RESULT_COMPUTED_FLAG) is False: - status = InvocationStatus.HIT - except Exception as e: - exc = e - status = InvocationStatus.ERROR - - finished_at = datetime.now(timezone.utc) - - if self._post_run_hooks: - record_id = ( - str(output_data.datagram_uuid) if output_data is not None else None - ) - payload = PostRunPayload( - record_id_hash=record_id, - tag=tag, - input=data, - output=output_data, - stats=RunStats( - duration_ms=(finished_at - started_at).total_seconds() * 1000, - status=status, - started_at=started_at, - finished_at=finished_at, - error=exc, - ), - pod=PodContext( - label=self.label, - pod_hash=self.content_hash().to_string(), - ), - ) - self._fire_post_run_hooks(payload) - - if exc is not None: - raise exc - return out_tag, output_data -``` - -You also need to add the missing import at the top of `cached_function_pod.py` — `DataExecutionLoggerProtocol` is already imported, but verify it is present: - -```python -from orcapod.protocols.observability_protocols import DataExecutionLoggerProtocol -``` - -- [ ] **Step 5: Run the cache hit tests — should pass** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestCacheHitStatus -x -q -``` - -Expected: PASS - -- [ ] **Step 6: Run full test suite for regressions** - -```bash -uv run pytest tests/test_core/function_pod/ -q 2>&1 | tail -10 -``` - -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/cached_function_pod.py tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "feat(cached_function_pod): override _invoke_with_hooks for cache hit status (ITL-523)" -``` - ---- - -## Task 6: `@function_pod` decorator + `orcapod.__init__` exports - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` (`function_pod` decorator signature) -- Modify: `src/orcapod/__init__.py` -- Modify: `tests/test_core/function_pod/test_post_run_hooks.py` (add `TestDecoratorConvenience`) - -- [ ] **Step 1: Add `TestDecoratorConvenience` to the test file** - -Append to `tests/test_core/function_pod/test_post_run_hooks.py`: - -```python -# --------------------------------------------------------------------------- -# 9. Decorator convenience -# --------------------------------------------------------------------------- - - -class TestDecoratorConvenience: - def test_decorator_post_run_hooks_fires_hook(self): - payloads: list[PostRunPayload] = [] - - @function_pod(output_keys="result", post_run_hooks=[payloads.append]) - def compute(x: int) -> int: - return x * 3 - - stream = compute.pod.process(_make_stream(n=2)) - list(stream.iter_data()) - - assert len(payloads) == 2 - assert all(p.stats.status == InvocationStatus.COMPUTED for p in payloads) - - def test_decorator_hookconfig_works(self): - payloads: list[PostRunPayload] = [] - - @function_pod( - output_keys="result", - post_run_hooks=[HookConfig(fn=payloads.append, on_error="log")], - ) - def compute2(x: int) -> int: - return x + 1 - - stream = compute2.pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - assert len(payloads) == 1 - - def test_public_api_imports(self): - import orcapod - assert hasattr(orcapod, "PostRunPayload") - assert hasattr(orcapod, "HookConfig") - assert hasattr(orcapod, "InvocationStatus") - assert hasattr(orcapod, "RunStats") - assert hasattr(orcapod, "PodContext") -``` - -- [ ] **Step 2: Run to confirm decorator test fails** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestDecoratorConvenience -x -q 2>&1 | head -15 -``` - -Expected: `TypeError: function_pod() got an unexpected keyword argument 'post_run_hooks'` - -- [ ] **Step 3: Update `function_pod` decorator signature** - -In `src/orcapod/core/function_pod.py`, find the `function_pod` function definition (around line 700). Add `post_run_hooks` parameter before `**kwargs`: - -```python -def function_pod( - output_keys: str | Sequence[str] | None = None, - function_name: str | None = None, - version: str = "v0.0", - label: str | None = None, - result_database: ArrowDatabaseProtocol | None = None, - pod_cache_database: ArrowDatabaseProtocol | None = None, - executor: DataFunctionExecutorProtocol | None = None, - post_run_hooks: Sequence[PostRunHook] | None = None, - **kwargs, -) -> Callable[..., CallableWithPodProtocol]: -``` - -Also add the `Sequence` import if not already present — it's already in the imports at the top of `function_pod.py` (`from collections.abc import Callable, Collection, Iterator, Sequence`), so no change needed there. - -- [ ] **Step 4: Register hooks in the decorator body** - -Inside the `decorator` function, after the `CachedFunctionPod` wrapping block (after the `if pod_cache_database is not None:` block, around line 760), add: - -```python - if post_run_hooks: - for hook in post_run_hooks: - pod.add_post_run_hook(hook) -``` - -- [ ] **Step 5: Update `src/orcapod/__init__.py` to export hook types** - -In `src/orcapod/__init__.py`, add after the existing imports: - -```python -from .hooks import ( - HookConfig, - InvocationStatus, - PodContext, - PostRunHook, - PostRunHookFn, - PostRunPayload, - RunStats, -) -``` - -And add these names to `__all__`: - -```python - # Post-run hook types (ITL-523) - "HookConfig", - "InvocationStatus", - "PodContext", - "PostRunHook", - "PostRunHookFn", - "PostRunPayload", - "RunStats", -``` - -- [ ] **Step 6: Run decorator and public API tests — should pass** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestDecoratorConvenience -x -q -``` - -Expected: PASS - -- [ ] **Step 7: Run the complete new test file** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py -v 2>&1 | tail -30 -``` - -Expected: all 10 test classes pass. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/core/function_pod.py src/orcapod/__init__.py tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "feat(function_pod): add post_run_hooks param to decorator and export public API (ITL-523)" -``` - ---- - -## Task 7: Full test suite + final commit - -- [ ] **Step 1: Run the entire test suite** - -```bash -uv run pytest tests/ -q 2>&1 | tail -20 -``` - -Expected: all tests pass, no regressions. - -- [ ] **Step 2: If any test fails, investigate and fix before proceeding** - -Common failure modes: -- Import errors: check that all new imports in `function_pod.py` and `cached_function_pod.py` are at the top of the file, not inside methods. -- `AttributeError: '_post_run_hooks'`: the `__init__` assignment in `_FunctionPodBase` may not have been saved — re-check step 4 of Task 2. -- `TypeError` in `_iter_data_concurrent`: verify both sync fallback and async gather paths were updated in Task 3. - -- [ ] **Step 3: Checkout the feature branch (if not already on it)** - -```bash -git branch --show-current -``` - -Expected: `eywalker/itl-523-function-pods-configurable-post-run-hook-invoked-after-every` - -If not on the branch: -```bash -git checkout -b eywalker/itl-523-function-pods-configurable-post-run-hook-invoked-after-every -``` - -- [ ] **Step 4: Push to remote** - -```bash -git push -u origin eywalker/itl-523-function-pods-configurable-post-run-hook-invoked-after-every -``` - -Expected: branch pushed, ready for PR. diff --git a/superpowers/plans/2026-07-14-itl-525-side-effect-pod.md b/superpowers/plans/2026-07-14-itl-525-side-effect-pod.md deleted file mode 100644 index 5802231c2..000000000 --- a/superpowers/plans/2026-07-14-itl-525-side-effect-pod.md +++ /dev/null @@ -1,2311 +0,0 @@ -# SideEffectPod Implementation Plan - -> **⚠ As-built notice (post-merge):** The invocation log design was updated during the PR -> review cycle. The code samples below in Tasks 7 and 8 still reference the original -> `status`/`error_message`-based schema and `get_records_with_column_value` completion check. -> The **actual implementation** differs: -> - Only **success** rows are written; failure writes nothing (so the next run retries). -> - **`record_id`** = `fip_hash.digest + b"::" + pod_content_hash.digest` (bytes, deterministic). -> - Completion check uses `get_record_by_id(table_path, record_id)` — no column scan. -> - Schema has no `status` or `error_message` columns. -> -> See `src/orcapod/side_effects.py` and `superpowers/specs/2026-07-14-itl-525-side-effect-pod-impl-design.md` for the authoritative as-built design. - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `SideEffectPod` as a first-class pipeline node for side effects (DB writes, logging, notifications) with pass-through output, deterministic invocation identity via `InvocationContext`, and optional idempotent completion tracking. - -**Architecture:** `SideEffectPod` is the user-facing pod (no DB); at pipeline compile time it is promoted to `SideEffectJobNode` (DB-backed), mirroring the `FunctionPod` → `FunctionJobNode` pattern. `run_id` is passed as a call-time keyword from orchestrators (never stored as node state). `InvocationHashConfig` is purely user-facing — no footprint in persistent storage. - -**Tech Stack:** PyArrow, asyncio.TaskGroup, `ArrowDatabaseProtocol`, `StreamBase`, `TraceableBase`, existing `PodInvocation` / `AbstractPipelineBase` / `BasicTrackerManager` patterns. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/side_effects.py` | **CREATE** | `InvocationHashConfig`, `SideEffectPodConfig`, `InvocationContext`, `SideEffectPodStream`, `SideEffectPod`, `SideEffectJobNode`, decorators | -| `src/orcapod/protocols/core_protocols/side_effect_pod.py` | **CREATE** | `SideEffectPodProtocol` | -| `src/orcapod/protocols/core_protocols/__init__.py` | **MODIFY** | re-export `SideEffectPodProtocol` | -| `src/orcapod/pipeline/pod_invocation.py` | **MODIFY** | add `SideEffectInvocation` | -| `src/orcapod/protocols/core_protocols/trackers.py` | **MODIFY** | add `record_side_effect_pod_invocation` to both protocols | -| `src/orcapod/core/tracker.py` | **MODIFY** | add `record_side_effect_pod_invocation` to `BasicTrackerManager` | -| `src/orcapod/pipeline/base.py` | **MODIFY** | add `record_side_effect_pod_invocation`, `side_effect_node_class` abstract property, `compile()` branch | -| `src/orcapod/pipeline/job.py` | **MODIFY** | `side_effect_node_class = SideEffectJobNode`, `_distribute_databases()` branch | -| `src/orcapod/protocols/node_protocols.py` | **MODIFY** | add `SideEffectNodeProtocol`, `is_side_effect_node()` | -| `src/orcapod/pipeline/sync_orchestrator.py` | **MODIFY** | add `elif is_side_effect_node` branch | -| `src/orcapod/pipeline/async_orchestrator.py` | **MODIFY** | add `elif is_side_effect_node` branch | -| `src/orcapod/__init__.py` | **MODIFY** | re-export public symbols | -| `tests/test_core/side_effect_pod/__init__.py` | **CREATE** | empty | -| `tests/test_core/side_effect_pod/test_side_effect_pod.py` | **CREATE** | T1–T18 | - ---- - -## Task 1: Types — `InvocationHashConfig`, `SideEffectPodConfig`, `InvocationContext` - -**Files:** -- Create: `src/orcapod/side_effects.py` -- Test: `tests/test_core/side_effect_pod/test_side_effect_pod.py` - -- [ ] **Step 1: Create test file and test directory** - -```bash -mkdir -p tests/test_core/side_effect_pod -touch tests/test_core/side_effect_pod/__init__.py -``` - -- [ ] **Step 2: Write failing tests for types** - -```python -# tests/test_core/side_effect_pod/test_side_effect_pod.py -from __future__ import annotations - -import pytest -import pyarrow as pa - -from orcapod.core.streams import ArrowTableStream -from orcapod.core.datagrams import Data, Tag - - -def _make_stream(n: int = 3) -> ArrowTableStream: - """Simple stream: tag=id (int), data=value (int).""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("value", pa.int64(), nullable=False), - ]) - table = pa.table( - {"id": list(range(n)), "value": list(range(n))}, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -# --------------------------------------------------------------------------- -# Task 1 tests -# --------------------------------------------------------------------------- - -class TestInvocationHashConfig: - def test_defaults(self): - from orcapod.side_effects import InvocationHashConfig - cfg = InvocationHashConfig() - assert cfg.encoding == "hex" - assert cfg.component_length is None - - def test_custom(self): - from orcapod.side_effects import InvocationHashConfig - cfg = InvocationHashConfig(encoding="base64", component_length=8) - assert cfg.encoding == "base64" - assert cfg.component_length == 8 - - def test_frozen(self): - from orcapod.side_effects import InvocationHashConfig - cfg = InvocationHashConfig() - with pytest.raises((AttributeError, TypeError)): - cfg.encoding = "base64" # type: ignore[misc] - - -class TestSideEffectPodConfig: - def test_defaults(self): - from orcapod.side_effects import SideEffectPodConfig - cfg = SideEffectPodConfig() - assert cfg.track_completion is True - assert cfg.drop_on_failure is True - assert cfg.on_error == "raise" - - def test_custom(self): - from orcapod.side_effects import SideEffectPodConfig - cfg = SideEffectPodConfig(track_completion=False, drop_on_failure=False) - assert cfg.track_completion is False - assert cfg.drop_on_failure is False -``` - -- [ ] **Step 3: Run to verify they fail** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestInvocationHashConfig tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectPodConfig -v -``` -Expected: FAIL with `ModuleNotFoundError` or `ImportError` - -- [ ] **Step 4: Create `src/orcapod/side_effects.py` with types** - -```python -# src/orcapod/side_effects.py -"""SideEffectPod — pass-through pipeline node for side effects. - -Provides ``SideEffectPod``, ``SideEffectPodStream``, ``SideEffectJobNode``, -``InvocationContext``, ``InvocationHashConfig``, ``SideEffectPodConfig``, -and the ``side_effect_pod``, ``sink_pod``, ``tap_pod`` decorators. -""" -from __future__ import annotations - -import asyncio -import base64 -import dataclasses -import datetime -import logging -from collections.abc import Callable, Collection, Iterator, Sequence -from typing import TYPE_CHECKING, Any, Literal - -from orcapod.core.base import TraceableBase -from orcapod.core.streams.base import StreamBase -from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.channels import ReadableChannel, WritableChannel - from orcapod.protocols.core_protocols import ( - DataProtocol, - StreamProtocol, - TagProtocol, - TrackerManagerProtocol, - ) - from orcapod.protocols.database_protocols import ArrowDatabaseProtocol - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol - from orcapod.types import ColumnConfig, ContentHash, Schema -else: - pa = LazyModule("pyarrow") - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# InvocationHashConfig -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass(frozen=True) -class InvocationHashConfig: - """Controls how ``InvocationContext.invocation_hash`` is serialized. - - Args: - encoding: Output encoding — ``"hex"`` (default), ``"base64"``, or - ``"binary"`` (falls back to hex in string contexts). - component_length: Bytes of raw digest to use per component. ``None`` - means full digest length. Applied identically to every - ``::``-separated component. - """ - - encoding: Literal["hex", "base64", "binary"] = "hex" - component_length: int | None = None - - -def _serialize_component(content_hash: ContentHash, config: InvocationHashConfig) -> str: - """Serialize one ``ContentHash`` component per ``InvocationHashConfig``. - - Args: - content_hash: The hash to serialize. - config: Encoding and truncation config. - - Returns: - A string representation of the (optionally truncated) digest. - """ - raw: bytes = content_hash.digest - if config.component_length is not None: - raw = raw[: config.component_length] - if config.encoding == "base64": - return base64.b64encode(raw).decode("ascii") - # "hex" and "binary" both produce hex strings in string contexts - return raw.hex() - - -# --------------------------------------------------------------------------- -# SideEffectPodConfig -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass(frozen=True) -class SideEffectPodConfig: - """Configuration for a ``SideEffectPod``. - - Args: - track_completion: If ``True`` (default), skip re-delivery for inputs - that previously completed successfully. - drop_on_failure: If ``True`` (default), drop rows whose delivery - raised an exception from the downstream output. - on_error: ``"raise"`` (default) re-raises delivery exceptions; - ``"log"`` logs at WARNING and continues. - hash_config: Controls encoding of ``InvocationContext.invocation_hash``. - """ - - track_completion: bool = True - drop_on_failure: bool = True - on_error: Literal["raise", "log"] = "raise" - hash_config: InvocationHashConfig = dataclasses.field( - default_factory=InvocationHashConfig - ) - - -# --------------------------------------------------------------------------- -# InvocationContext -# --------------------------------------------------------------------------- - - -class InvocationContext: - """Per-invocation context passed to every side-effect pod function. - - Carries the deterministic ``invocation_hash`` string and metadata about - the current delivery. ``format_id()`` re-serializes the hash with a - caller-supplied ``InvocationHashConfig`` without recomputing. - - Public fields are read-only by convention (no public setters). - - Args: - invocation_hash: Serialized compound hash string. - pod_name: ``pod.label`` of the invoking pod. - pod_content_hash: ``pod.content_hash().to_string()``. - pipeline_run_id: The current pipeline run identifier, or ``None`` - for standalone / lazy pipelines. - """ - - def __init__( - self, - invocation_hash: str, - pod_name: str, - pod_content_hash: str, - pipeline_run_id: str | None, - _pipeline_hash_ch: ContentHash, - _full_input_packet_hash_ch: ContentHash, - _hash_config: InvocationHashConfig, - _track_completion: bool, - ) -> None: - self.invocation_hash = invocation_hash - self.pod_name = pod_name - self.pod_content_hash = pod_content_hash - self.pipeline_run_id = pipeline_run_id - self._pipeline_hash_ch = _pipeline_hash_ch - self._full_input_packet_hash_ch = _full_input_packet_hash_ch - self._hash_config = _hash_config - self._track_completion = _track_completion - - def format_id(self, config: InvocationHashConfig | None = None) -> str: - """Return ``'orcapod-{hash}'`` with an optional format override. - - Re-serializes from the stored raw ``ContentHash`` components — no - recomputation. Uses ``config`` if supplied, otherwise the pod's own - ``InvocationHashConfig``. - - Args: - config: Optional encoding/truncation override. - - Returns: - A string of the form ``"orcapod-{component1}::{component2}"`` - (two components when ``track_completion=True``) or - ``"orcapod-{c1}::{c2}::{run_id}"`` (three components when - ``track_completion=False`` and ``pipeline_run_id`` is not ``None``). - """ - cfg = config or self._hash_config - c1 = _serialize_component(self._pipeline_hash_ch, cfg) - c2 = _serialize_component(self._full_input_packet_hash_ch, cfg) - if not self._track_completion and self.pipeline_run_id is not None: - parts = f"{c1}::{c2}::{self.pipeline_run_id}" - else: - parts = f"{c1}::{c2}" - return f"orcapod-{parts}" -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestInvocationHashConfig tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectPodConfig -v -``` -Expected: PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/side_effects.py tests/test_core/side_effect_pod/__init__.py tests/test_core/side_effect_pod/test_side_effect_pod.py -git commit -m "feat(side-effects): add InvocationHashConfig, SideEffectPodConfig, InvocationContext" -``` - ---- - -## Task 2: `SideEffectPodProtocol` - -**Files:** -- Create: `src/orcapod/protocols/core_protocols/side_effect_pod.py` -- Modify: `src/orcapod/protocols/core_protocols/__init__.py` - -- [ ] **Step 1: Create protocol file** - -```python -# src/orcapod/protocols/core_protocols/side_effect_pod.py -"""SideEffectPodProtocol — protocol for side-effect pods.""" -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol, runtime_checkable - -from orcapod.protocols.hashing_protocols import PipelineElementProtocol - -if TYPE_CHECKING: - from orcapod.protocols.core_protocols.pod import ArgumentGroup, PodProtocol - from orcapod.protocols.core_protocols.streams import StreamProtocol - from orcapod.side_effects import SideEffectPodConfig, SideEffectPodStream - from orcapod.types import ColumnConfig, Schema - - -@runtime_checkable -class SideEffectPodProtocol(PipelineElementProtocol, Protocol): - """Protocol for side-effect pods. - - A side-effect pod wraps a ``(data: T, ctx: InvocationContext) -> None`` - callable. Its ``process()`` returns a pass-through stream. Output schema - equals input schema. - """ - - @property - def pod_config(self) -> "SideEffectPodConfig": - """Pod-level configuration.""" - ... - - def process( - self, *streams: "StreamProtocol", label: str | None = None - ) -> "SideEffectPodStream": - """Invoke the pod on input streams, returning a pass-through stream.""" - ... - - def output_schema( - self, - *streams: "StreamProtocol", - columns: "ColumnConfig | dict[str, object] | None" = None, - all_info: bool = False, - ) -> "tuple[Schema, Schema]": - """Return the input stream's schema unchanged (pass-through).""" - ... - - def argument_symmetry( - self, streams: "Collection[StreamProtocol]" - ) -> "ArgumentGroup": ... -``` - -- [ ] **Step 2: Re-export from `core_protocols/__init__.py`** - -Open `src/orcapod/protocols/core_protocols/__init__.py` and add: - -```python -from .side_effect_pod import SideEffectPodProtocol -``` - -And add `"SideEffectPodProtocol"` to `__all__`. - -- [ ] **Step 3: Verify import** - -```bash -uv run python -c "from orcapod.protocols.core_protocols import SideEffectPodProtocol; print('ok')" -``` -Expected: `ok` - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/protocols/core_protocols/side_effect_pod.py src/orcapod/protocols/core_protocols/__init__.py -git commit -m "feat(protocols): add SideEffectPodProtocol" -``` - ---- - -## Task 3: `SideEffectInvocation` - -**Files:** -- Modify: `src/orcapod/pipeline/pod_invocation.py` - -- [ ] **Step 1: Write failing test** - -Add to `tests/test_core/side_effect_pod/test_side_effect_pod.py`: - -```python -class TestSideEffectInvocation: - def test_construction(self): - from orcapod.pipeline.pod_invocation import SideEffectInvocation - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - - calls = [] - def fn(data, ctx): - calls.append(data) - - pod = SideEffectPod(fn) - stream = _make_stream() - inv = SideEffectInvocation(pod=pod, input_streams=(stream,)) - assert inv.pod is pod - assert inv.input_streams == (stream,) - - def test_requires_exactly_one_stream(self): - from orcapod.pipeline.pod_invocation import SideEffectInvocation - from orcapod.side_effects import SideEffectPod - - def fn(data, ctx): pass - pod = SideEffectPod(fn) - stream = _make_stream() - - with pytest.raises(ValueError): - SideEffectInvocation(pod=pod, input_streams=()) - - with pytest.raises(ValueError): - SideEffectInvocation(pod=pod, input_streams=(stream, stream)) -``` - -- [ ] **Step 2: Run to verify failure** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectInvocation -v -``` -Expected: FAIL (`ImportError` on `SideEffectInvocation`) - -- [ ] **Step 3: Add `SideEffectInvocation` to `pod_invocation.py`** - -Open `src/orcapod/pipeline/pod_invocation.py` and append after `OperatorInvocation`: - -```python -class SideEffectInvocation(PodInvocation): - """Invocation of a side-effect pod against exactly one input stream. - - Args: - pod: A ``SideEffectPodProtocol`` instance. - input_streams: Tuple with exactly one stream. - label: Optional display label. - - Raises: - ValueError: If ``input_streams`` does not contain exactly one element. - """ - - def __init__( - self, - pod: Any, - input_streams: "tuple[StreamProtocol, ...]", - label: str | None = None, - ) -> None: - if len(input_streams) != 1: - raise ValueError( - f"SideEffectInvocation requires exactly 1 input stream; " - f"got {len(input_streams)}." - ) - super().__init__(pod=pod, input_streams=input_streams, label=label) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectInvocation -v -``` -Expected: PASS (note: `SideEffectPod` may not exist yet — acceptable FAIL with `ImportError` on `SideEffectPod` at this stage; the SideEffectInvocation logic itself is correct) - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/pipeline/pod_invocation.py -git commit -m "feat(pipeline): add SideEffectInvocation to pod_invocation" -``` - ---- - -## Task 4: Tracker chain — `record_side_effect_pod_invocation` - -**Files:** -- Modify: `src/orcapod/protocols/core_protocols/trackers.py` -- Modify: `src/orcapod/core/tracker.py` -- Modify: `src/orcapod/pipeline/base.py` - -- [ ] **Step 1: Add to `TrackerProtocol` and `TrackerManagerProtocol`** - -Open `src/orcapod/protocols/core_protocols/trackers.py`. - -Add the following method to **`TrackerProtocol`** after `record_function_pod_invocation`: - -```python - def record_side_effect_pod_invocation( - self, - pod: Any, - input_stream: StreamProtocol, - label: str | None = None, - ) -> None: - """Record a side-effect pod invocation in the computational graph. - - Args: - pod: The side-effect pod being invoked. - input_stream: The upstream stream. - label: Optional display label. - """ - ... -``` - -Add the same method to **`TrackerManagerProtocol`** after `record_function_pod_invocation`: - -```python - def record_side_effect_pod_invocation( - self, - pod: Any, - input_stream: StreamProtocol, - label: str | None = None, - ) -> None: - """Record a side-effect pod invocation in all active trackers. - - Args: - pod: The side-effect pod to record. - input_stream: The upstream stream. - label: Optional display label. - """ - ... -``` - -Note: use `Any` for `pod` type to avoid circular imports. Add `from typing import Any` if not already imported. - -- [ ] **Step 2: Add to `BasicTrackerManager` in `core/tracker.py`** - -Open `src/orcapod/core/tracker.py` and add after `record_function_pod_invocation`: - -```python - def record_side_effect_pod_invocation( - self, - pod: Any, - input_stream: cp.StreamProtocol, - label: str | None = None, - ) -> None: - """Record a side-effect pod invocation in all active trackers.""" - for tracker in self.get_active_trackers(): - tracker.record_side_effect_pod_invocation(pod, input_stream, label=label) -``` - -Add `from typing import Any` at the top of the file if not already present. - -- [ ] **Step 3: Add `record_side_effect_pod_invocation` to `AbstractPipelineBase`** - -Open `src/orcapod/pipeline/base.py`. - -Add the following import at the top with the other invocation imports: -```python -from orcapod.pipeline.pod_invocation import ( - FunctionInvocation, - OperatorInvocation, - PodInvocation, - SideEffectInvocation, # add this -) -``` - -Add the new method after `record_operator_pod_invocation`: - -```python - def record_side_effect_pod_invocation( - self, - pod: Any, - input_stream: "cp.StreamProtocol", - label: str | None = None, - ) -> None: - """Record a side-effect pod invocation into the graph. - - Args: - pod: The side-effect pod being invoked. - input_stream: The upstream stream. - label: Optional display label for the resulting compiled node. - """ - self._record_invocation( - SideEffectInvocation(pod=pod, input_streams=(input_stream,), label=label) - ) -``` - -Add `from typing import Any` if not already imported. - -- [ ] **Step 4: Verify no import errors** - -```bash -uv run python -c "from orcapod.pipeline.base import AbstractPipelineBase; print('ok')" -uv run python -c "from orcapod.core.tracker import BasicTrackerManager; print('ok')" -``` -Expected: both print `ok` - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/protocols/core_protocols/trackers.py src/orcapod/core/tracker.py src/orcapod/pipeline/base.py -git commit -m "feat(tracker): add record_side_effect_pod_invocation to tracker chain" -``` - ---- - -## Task 5: `SideEffectPod`, `SideEffectPodStream`, and decorators - -**Files:** -- Modify: `src/orcapod/side_effects.py` -- Test: `tests/test_core/side_effect_pod/test_side_effect_pod.py` - -- [ ] **Step 1: Write failing tests T1–T4, T8–T10, T16–T18** - -Add to `tests/test_core/side_effect_pod/test_side_effect_pod.py`: - -```python -# --------------------------------------------------------------------------- -# Task 5 tests — standalone / lazy mode (no DB) -# --------------------------------------------------------------------------- - - -class TestSideEffectPodStandalone: - """T1–T4, T8–T10: standalone execution via SideEffectPodStream.""" - - def test_t1_passthrough_drop_on_failure_false(self): - """T1: All rows emitted when drop_on_failure=False and no errors.""" - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - - calls = [] - def fn(data, ctx): - calls.append(dict(data)) - - pod = SideEffectPod(fn, config=SideEffectPodConfig(drop_on_failure=False)) - stream = _make_stream(3) - out = list(pod.process(stream).iter_data()) - - assert len(out) == 3 - assert len(calls) == 3 - - def test_t2_passthrough_drop_on_failure_true_no_errors(self): - """T2: All rows emitted when drop_on_failure=True and no errors.""" - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - - calls = [] - def fn(data, ctx): - calls.append(dict(data)) - - pod = SideEffectPod(fn) # default: drop_on_failure=True - stream = _make_stream(3) - out = list(pod.process(stream).iter_data()) - - assert len(out) == 3 - assert len(calls) == 3 - - def test_t3_invocation_context_always_passed(self): - """T3: InvocationContext always constructed and passed.""" - from orcapod.side_effects import SideEffectPod, InvocationContext - - received = [] - def fn(data, ctx): - received.append(ctx) - - pod = SideEffectPod(fn) - stream = _make_stream(1) - list(pod.process(stream).iter_data()) - - assert len(received) == 1 - ctx = received[0] - assert isinstance(ctx, InvocationContext) - assert isinstance(ctx.invocation_hash, str) - assert len(ctx.invocation_hash) > 0 - assert ctx.format_id().startswith("orcapod-") - - def test_t4_invocation_context_ignored_by_callee(self): - """T4: Pod works fine when callee ignores ctx.""" - from orcapod.side_effects import SideEffectPod - - calls = [] - def fn(data, _ctx): - calls.append(True) - - pod = SideEffectPod(fn) - stream = _make_stream(2) - out = list(pod.process(stream).iter_data()) - - assert len(out) == 2 - assert len(calls) == 2 - - def test_t8_on_error_raise(self): - """T8: on_error='raise' propagates the exception.""" - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - - def fn(data, ctx): - raise RuntimeError("boom") - - pod = SideEffectPod(fn, config=SideEffectPodConfig(on_error="raise")) - stream = _make_stream(1) - - with pytest.raises(RuntimeError, match="boom"): - list(pod.process(stream).iter_data()) - - def test_t9_on_error_log_drop_on_failure_true(self): - """T9: on_error='log' + drop_on_failure=True drops failed rows.""" - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - - def fn(data, ctx): - raise RuntimeError("oops") - - pod = SideEffectPod( - fn, - config=SideEffectPodConfig(on_error="log", drop_on_failure=True), - ) - stream = _make_stream(3) - out = list(pod.process(stream).iter_data()) - - assert len(out) == 0 # all rows dropped - - def test_t10_on_error_log_drop_on_failure_false(self): - """T10: on_error='log' + drop_on_failure=False passes through despite failure.""" - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - - def fn(data, ctx): - raise RuntimeError("oops") - - pod = SideEffectPod( - fn, - config=SideEffectPodConfig(on_error="log", drop_on_failure=False), - ) - stream = _make_stream(3) - out = list(pod.process(stream).iter_data()) - - assert len(out) == 3 # all rows still emitted - - -class TestDecorators: - """T16–T18: @sink_pod, @tap_pod, @side_effect_pod.""" - - def test_t16_sink_pod(self): - """T16: @sink_pod sets track_completion=True, drop_on_failure=True.""" - from orcapod.side_effects import sink_pod, SideEffectPod - - @sink_pod - def my_sink(data, ctx): - pass - - assert isinstance(my_sink, SideEffectPod) - assert my_sink.pod_config.track_completion is True - assert my_sink.pod_config.drop_on_failure is True - - def test_t17_tap_pod(self): - """T17: @tap_pod sets track_completion=False, drop_on_failure=False.""" - from orcapod.side_effects import tap_pod, SideEffectPod - - @tap_pod - def my_tap(data, ctx): - pass - - assert isinstance(my_tap, SideEffectPod) - assert my_tap.pod_config.track_completion is False - assert my_tap.pod_config.drop_on_failure is False - - def test_t18_side_effect_pod_config_combinations(self): - """T18: @side_effect_pod(config=...) all four combinations.""" - from orcapod.side_effects import side_effect_pod, SideEffectPodConfig - - for tc in [True, False]: - for dof in [True, False]: - cfg = SideEffectPodConfig(track_completion=tc, drop_on_failure=dof) - - @side_effect_pod(config=cfg) - def fn(data, ctx): - pass - - assert fn.pod_config.track_completion is tc - assert fn.pod_config.drop_on_failure is dof - - def test_sink_pod_parameterised(self): - """@sink_pod(config=...) with explicit config override.""" - from orcapod.side_effects import sink_pod, SideEffectPodConfig - - cfg = SideEffectPodConfig(on_error="log") - - @sink_pod(config=cfg) - def my_sink(data, ctx): - pass - - assert my_sink.pod_config.on_error == "log" - assert my_sink.pod_config.track_completion is True - assert my_sink.pod_config.drop_on_failure is True -``` - -- [ ] **Step 2: Run to verify failure** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectPodStandalone tests/test_core/side_effect_pod/test_side_effect_pod.py::TestDecorators -v -``` -Expected: FAIL (`ImportError` on `SideEffectPod`) - -- [ ] **Step 3: Implement `SideEffectPodStream`, `SideEffectPod`, and decorators in `side_effects.py`** - -Append the following to `src/orcapod/side_effects.py` (after the `InvocationContext` class): - -```python -# --------------------------------------------------------------------------- -# SideEffectPodStream -# --------------------------------------------------------------------------- - - -class SideEffectPodStream(StreamBase): - """Pass-through stream returned by ``SideEffectPod.process()`` in standalone mode. - - Iterates the upstream stream and calls the side-effect function per row. - No invocation log is written in standalone mode (``pipeline_run_id=None``). - """ - - def __init__( - self, - side_effect_pod: SideEffectPod, - input_stream: StreamProtocol, - **kwargs: Any, - ) -> None: - self._pod = side_effect_pod - self._input_stream = input_stream - super().__init__(**kwargs) - - @property - def producer(self): # type: ignore[override] - return self._pod - - @property - def upstreams(self) -> tuple[StreamProtocol, ...]: - return (self._input_stream,) - - def identity_structure(self) -> Any: - return (self._pod, self._pod.argument_symmetry((self._input_stream,))) - - def pipeline_identity_structure(self) -> Any: - return self.identity_structure() - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - return self._input_stream.output_schema(columns=columns, all_info=all_info) - - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - return self._input_stream.keys(columns=columns, all_info=all_info) - - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - for tag, data in self._input_stream.iter_data(): - result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self._pod.pipeline_hash(), - pod_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=None, - arrow_hasher=self._pod.data_context.arrow_hasher, - ) - if result is not None: - yield result - - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table: - from orcapod.utils import arrow_utils - - tag_tables = [] - data_tables = [] - for tag, data in self.iter_data(): - tag_tables.append(tag.as_table(columns={"system_tags": True})) - data_tables.append(data.as_table(columns={"source": True})) - if not tag_tables: - # Return an empty table with the correct schema - tag_schema, data_schema = self.output_schema( - columns={"system_tags": True, "source": True} - ) - tc = self._pod.data_context.type_converter - fields = {} - for name, py_type in {**tag_schema, **data_schema}.items(): - fields[name] = pa.array( - [], type=tc.python_type_to_arrow_type(py_type) - ) - return pa.table(fields) - combined_tags = pa.concat_tables(tag_tables) - combined_data = pa.concat_tables(data_tables) - return arrow_utils.hstack_tables(combined_tags, combined_data) - - -# --------------------------------------------------------------------------- -# Shared row execution helper -# --------------------------------------------------------------------------- - - -def _execute_side_effect_row( - *, - fn: Callable, - tag: TagProtocol, - data: DataProtocol, - pod_config: SideEffectPodConfig, - pipeline_hash_ch: ContentHash, - pod_content_hash_str: str, - pod_name: str, - run_id: str | None, - arrow_hasher: Any, - pipeline_database: ArrowDatabaseProtocol | None = None, - table_path: tuple[str, ...] | None = None, -) -> tuple[TagProtocol, DataProtocol] | None: - """Execute delivery for one (tag, data) row. - - Args: - fn: The side-effect callable ``(data, ctx) -> None``. - tag: Tag for this row. - data: Data for this row. - pod_config: Pod-level configuration. - pipeline_hash_ch: Pipeline hash of the node (for invocation_hash c1). - pod_content_hash_str: String form of the pod's content hash. - pod_name: Label of the pod. - run_id: Pipeline run identifier (or ``None`` in standalone mode). - arrow_hasher: The ``arrow_hasher`` from the pod's data context. - pipeline_database: Attached DB (or ``None`` for standalone mode). - table_path: Path tuple for the invocation log table. - - Returns: - ``(tag, data)`` to emit downstream, or ``None`` to drop the row. - """ - from orcapod.utils import arrow_utils - - # 1. Compute full_input_packet_hash over all four column groups. - tag_table = tag.as_table(columns={"system_tags": True}) - data_table = data.as_table(columns={"source": True}) - full_table = arrow_utils.hstack_tables(tag_table, data_table) - fip_hash: ContentHash = arrow_hasher.hash_table(full_table) - fip_hash_str = fip_hash.to_string() - - # 2. Serialize invocation_hash. - cfg = pod_config.hash_config - c1 = _serialize_component(pipeline_hash_ch, cfg) - c2 = _serialize_component(fip_hash, cfg) - if not pod_config.track_completion and run_id is not None: - inv_hash = f"{c1}::{c2}::{run_id}" - else: - inv_hash = f"{c1}::{c2}" - - # ⚠ As-built: the completion check and write logic below was redesigned post-merge. - # Actual implementation uses a deterministic record_id + get_record_by_id lookup. - # Only success rows are written; failure writes nothing so the next run retries. - # See src/orcapod/side_effects.py and the spec for the authoritative design. - - # 3. Deterministic record_id for completion lookup and write. - record_id = fip_hash.digest + b"::" + pod_content_hash.digest - - # 4. Completion check — O(1) lookup by record_id; no column scan. - if pod_config.track_completion and pipeline_database is not None and table_path is not None: - prior = pipeline_database.get_record_by_id(table_path, record_id) - if prior is not None: - return (tag, data) # already completed — re-emit without re-delivery - - # 5. Build InvocationContext (always). - ctx = InvocationContext( - invocation_hash=inv_hash, - pod_name=pod_name, - pod_content_hash=pod_content_hash.to_string(), - pipeline_run_id=run_id, - _pipeline_hash_ch=pipeline_hash_ch, - _full_input_packet_hash_ch=fip_hash, - _hash_config=cfg, - _track_completion=pod_config.track_completion, - ) - - # 6. Call user function. - try: - fn(data, ctx) - if pipeline_database is not None and table_path is not None: - _write_invocation_row( - pipeline_database=pipeline_database, - table_path=table_path, - record_id=record_id, - fip_hash_str=fip_hash.to_string(), - pod_content_hash_str=pod_content_hash.to_string(), - run_id=run_id, - ) - return (tag, data) - except Exception as exc: - # No DB write on failure — absence of record means the next run retries. - if pod_config.on_error == "raise": - raise - logger.warning( - "SideEffectPod %r delivery failed: %s", pod_name, exc, exc_info=True - ) - if pod_config.drop_on_failure: - return None - return (tag, data) - - -def _write_invocation_row( - *, - pipeline_database: ArrowDatabaseProtocol, - table_path: tuple[str, ...], - fip_hash_str: str, - pod_content_hash_str: str, - run_id: str | None, - status: str, - error_message: str | None, -) -> None: - """Write one row to the side-effect invocation log table.""" - record = pa.table( - { - "full_input_packet_hash": pa.array( - [fip_hash_str], type=pa.large_string() - ), - "pod_content_hash": pa.array( - [pod_content_hash_str], type=pa.large_string() - ), - "pipeline_run_id": pa.array( - [run_id], type=pa.large_string() - ), - "executed_at": pa.array( - [datetime.datetime.now(datetime.timezone.utc)], - type=pa.timestamp("us", tz="UTC"), - ), - "status": pa.array([status], type=pa.large_string()), - "error_message": pa.array( - [error_message], type=pa.large_string() - ), - } - ) - pipeline_database.add_records( - table_path, - record, - record_id_column=None, - skip_duplicates=False, - ) - - -# --------------------------------------------------------------------------- -# SideEffectPod -# --------------------------------------------------------------------------- - - -class SideEffectPod(TraceableBase): - """A pipeline node whose primary purpose is a side effect. - - Wraps a ``(data: T, ctx: InvocationContext) -> None`` callable. - ``InvocationContext`` is always constructed and passed — it is part of - the function contract. Callees that do not need it may ignore it (name - the parameter ``_ctx`` by convention). - - Returns a pass-through stream. When ``drop_on_failure=True``, only - successfully-delivered rows flow downstream. - - In standalone mode (no ``PipelineJob``), executes row-by-row via - ``SideEffectPodStream`` with no invocation logging. - - In pipeline mode, promoted to ``SideEffectJobNode`` at compile time, - which adds DB-backed invocation logging and completion tracking. - - Args: - fn: A callable ``(data, ctx: InvocationContext) -> None``. - config: Pod-level configuration. Defaults to ``SideEffectPodConfig()``. - tracker_manager: Optional tracker manager override. - label: Optional display label. - data_context: Optional data context override. - """ - - def __init__( - self, - fn: Callable, - config: SideEffectPodConfig | None = None, - tracker_manager: TrackerManagerProtocol | None = None, - label: str | None = None, - data_context: Any = None, - ) -> None: - super().__init__(label=label, data_context=data_context) - self._fn = fn - self._pod_config = config or SideEffectPodConfig() - self.tracker_manager = tracker_manager or DEFAULT_TRACKER_MANAGER - - @property - def pod_config(self) -> SideEffectPodConfig: - """Pod-level configuration.""" - return self._pod_config - - def computed_label(self) -> str | None: - """Use the callable's ``__name__`` as the default label.""" - return getattr(self._fn, "__name__", None) - - def identity_structure(self) -> Any: - return ("SideEffectPod", self._fn, self._pod_config) - - def pipeline_identity_structure(self) -> Any: - return self.identity_structure() - - @property - def uri(self) -> tuple[str, ...]: - """Canonical URI for this pod.""" - module = getattr(self._fn, "__module__", "unknown") - name = getattr(self._fn, "__qualname__", getattr(self._fn, "__name__", "unknown")) - return (module, name) - - def argument_symmetry(self, streams: Collection[StreamProtocol]) -> Any: - """Single ordered input — return as an ordered tuple.""" - return tuple(streams) - - def output_schema( - self, - *streams: StreamProtocol, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return the input stream's schema unchanged (pass-through). - - Args: - *streams: Exactly one input stream. - columns: Optional column config. - all_info: Include all metadata columns. - - Returns: - The input stream's ``(tag_schema, data_schema)`` unchanged. - """ - if len(streams) != 1: - raise ValueError( - f"SideEffectPod expects exactly 1 input stream; got {len(streams)}." - ) - return streams[0].output_schema(columns=columns, all_info=all_info) - - def process( - self, *streams: StreamProtocol, label: str | None = None - ) -> SideEffectPodStream: - """Invoke the side-effect pod on the input stream. - - Registers a ``SideEffectInvocation`` with the tracker manager (if - inside a ``with PipelineJob():`` block), then returns a - ``SideEffectPodStream`` for standalone / lazy execution. - - Args: - *streams: Exactly one input stream. - label: Optional label for the compiled node. - - Returns: - A ``SideEffectPodStream``. - """ - if len(streams) != 1: - raise ValueError( - f"SideEffectPod.process() expects exactly 1 stream; got {len(streams)}." - ) - input_stream = streams[0] - self.tracker_manager.record_side_effect_pod_invocation( - self, input_stream, label=label - ) - return SideEffectPodStream( - side_effect_pod=self, - input_stream=input_stream, - label=label, - ) - - def __call__( - self, *streams: StreamProtocol, label: str | None = None - ) -> SideEffectPodStream: - """Convenience alias for ``process``.""" - return self.process(*streams, label=label) - - -# --------------------------------------------------------------------------- -# Decorators -# --------------------------------------------------------------------------- - - -def side_effect_pod( - fn: Callable | None = None, - *, - config: SideEffectPodConfig | None = None, -) -> SideEffectPod | Callable[[Callable], SideEffectPod]: - """Decorator that wraps a callable as a ``SideEffectPod``. - - Supports both bare (``@side_effect_pod``) and parameterised - (``@side_effect_pod(config=...)``) usage. - - Args: - fn: The callable to wrap (when used as a bare decorator). - config: Optional ``SideEffectPodConfig`` to apply. - - Returns: - A ``SideEffectPod`` (bare usage) or a decorator (parameterised usage). - """ - def _wrap(f: Callable) -> SideEffectPod: - return SideEffectPod(f, config=config) - - if fn is not None: - return _wrap(fn) - return _wrap - - -def sink_pod( - fn: Callable | None = None, - *, - config: SideEffectPodConfig | None = None, -) -> SideEffectPod | Callable[[Callable], SideEffectPod]: - """Decorator preset: ``track_completion=True``, ``drop_on_failure=True``. - - Caller-supplied ``config`` fields override the presets. Supports both - bare (``@sink_pod``) and parameterised (``@sink_pod(config=...)``) usage. - - Args: - fn: The callable to wrap (bare usage). - config: Optional config override. - - Returns: - A ``SideEffectPod`` or decorator. - """ - preset = SideEffectPodConfig(track_completion=True, drop_on_failure=True) - effective_config = _merge_config(preset, config) - - def _wrap(f: Callable) -> SideEffectPod: - return SideEffectPod(f, config=effective_config) - - if fn is not None: - return _wrap(fn) - return _wrap - - -def tap_pod( - fn: Callable | None = None, - *, - config: SideEffectPodConfig | None = None, -) -> SideEffectPod | Callable[[Callable], SideEffectPod]: - """Decorator preset: ``track_completion=False``, ``drop_on_failure=False``. - - Caller-supplied ``config`` fields override the presets. Supports both - bare (``@tap_pod``) and parameterised (``@tap_pod(config=...)``) usage. - - Args: - fn: The callable to wrap (bare usage). - config: Optional config override. - - Returns: - A ``SideEffectPod`` or decorator. - """ - preset = SideEffectPodConfig(track_completion=False, drop_on_failure=False) - effective_config = _merge_config(preset, config) - - def _wrap(f: Callable) -> SideEffectPod: - return SideEffectPod(f, config=effective_config) - - if fn is not None: - return _wrap(fn) - return _wrap - - -def _merge_config( - preset: SideEffectPodConfig, - override: SideEffectPodConfig | None, -) -> SideEffectPodConfig: - """Merge *preset* with caller-supplied *override*. - - Non-default fields in *override* win over the preset. - - Args: - preset: The decorator's pre-configured defaults. - override: Optional caller-supplied config. - - Returns: - A merged ``SideEffectPodConfig``. - """ - if override is None: - return preset - default = SideEffectPodConfig() - return SideEffectPodConfig( - track_completion=( - override.track_completion - if override.track_completion != default.track_completion - else preset.track_completion - ), - drop_on_failure=( - override.drop_on_failure - if override.drop_on_failure != default.drop_on_failure - else preset.drop_on_failure - ), - on_error=( - override.on_error - if override.on_error != default.on_error - else preset.on_error - ), - hash_config=( - override.hash_config - if override.hash_config != default.hash_config - else preset.hash_config - ), - ) -``` - -- [ ] **Step 4: Run standalone tests** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectPodStandalone tests/test_core/side_effect_pod/test_side_effect_pod.py::TestDecorators -v -``` -Expected: PASS - -- [ ] **Step 5: Run Task 3 tests now that `SideEffectPod` exists** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectInvocation -v -``` -Expected: PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/side_effects.py -git commit -m "feat(side-effects): add SideEffectPod, SideEffectPodStream, decorators" -``` - ---- - -## Task 6: `SideEffectNodeProtocol` + `is_side_effect_node()` - -**Files:** -- Modify: `src/orcapod/protocols/node_protocols.py` - -- [ ] **Step 1: Add `SideEffectNodeProtocol` and TypeGuard** - -Open `src/orcapod/protocols/node_protocols.py`. After `OperatorNodeProtocol`, add: - -```python -@runtime_checkable -class SideEffectNodeProtocol(Protocol): - """Protocol for side-effect nodes in orchestrated execution.""" - - node_type: str - - def execute( - self, - input_stream: "StreamProtocol", - *, - observer: "ExecutionObserverProtocol | None" = None, - run_id: str | None = None, - ) -> "list[tuple[TagProtocol, DataProtocol]]": ... - - async def async_execute( - self, - inputs: "Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]]", - output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", - *, - observer: "ExecutionObserverProtocol | None" = None, - run_id: str | None = None, - ) -> None: ... - - def attach_databases( - self, - pipeline_database: "ArrowDatabaseProtocol | None" = None, - ) -> None: ... -``` - -Then add the TypeGuard after `is_operator_node`: - -```python -def is_side_effect_node(node: "GraphNode") -> "TypeGuard[SideEffectNodeProtocol]": - """Check if a node is a side-effect node.""" - return node.node_type == "side_effect" -``` - -- [ ] **Step 2: Verify import** - -```bash -uv run python -c "from orcapod.protocols.node_protocols import is_side_effect_node; print('ok')" -``` -Expected: `ok` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/protocols/node_protocols.py -git commit -m "feat(protocols): add SideEffectNodeProtocol and is_side_effect_node TypeGuard" -``` - ---- - -## Task 7: `SideEffectJobNode` (sync execution + DB) - -**Files:** -- Modify: `src/orcapod/side_effects.py` -- Test: `tests/test_core/side_effect_pod/test_side_effect_pod.py` - -- [ ] **Step 1: Write failing DB tests T5–T7, T11–T12** - -Add to `tests/test_core/side_effect_pod/test_side_effect_pod.py`: - -```python -# --------------------------------------------------------------------------- -# Task 7 tests — DB-backed execution via SideEffectJobNode -# --------------------------------------------------------------------------- - - -def _make_in_memory_db(): - """Return a fresh in-memory ArrowDatabase.""" - from orcapod.databases.in_memory import InMemoryDatabase - return InMemoryDatabase() - - -class TestSideEffectJobNodeSync: - """T5–T7, T11–T12: DB-backed sync execution.""" - - def _make_node_with_db(self, fn, config=None): - """Helper: build a SideEffectJobNode with an in-memory DB attached.""" - from orcapod.side_effects import SideEffectPod, SideEffectJobNode - - pod = SideEffectPod(fn, config=config) - stream = _make_stream(3) - node = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - db = _make_in_memory_db() - node.attach_databases(pipeline_database=db) - return node, stream, db - - def test_t5_invocation_log_written_on_success(self): - """T5: DB row written with status='success'.""" - import polars as pl - from orcapod.side_effects import SideEffectPod, SideEffectJobNode - - calls = [] - def fn(data, ctx): - calls.append(True) - - node, stream, db = self._make_node_with_db(fn) - results = node.execute(stream) - assert len(results) == 3 - assert len(calls) == 3 - - # Read log table - table_path = (node.pipeline_hash().to_string(), "side_effect_invocations") - records = db.get_all_records(table_path) - assert records is not None - df = pl.from_arrow(records) - assert len(df) == 3 - assert all(df["status"] == "success") - assert "full_input_packet_hash" in df.columns - assert "invocation_hash" not in df.columns # never stored - - def test_t6_track_completion_skips_on_rerun(self): - """T6: Second run skips re-delivery; skipped row still emitted.""" - import polars as pl - from orcapod.side_effects import SideEffectPod, SideEffectJobNode, SideEffectPodConfig - - calls = [] - def fn(data, ctx): - calls.append(True) - - cfg = SideEffectPodConfig(track_completion=True) - pod = SideEffectPod(fn, config=cfg) - stream = _make_stream(2) - db = _make_in_memory_db() - - node1 = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=db) - results1 = node1.execute(stream) - assert len(results1) == 2 - assert len(calls) == 2 - - # Second run with same DB - node2 = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=db) - results2 = node2.execute(stream) - assert len(results2) == 2 # rows still emitted (pass-through) - assert len(calls) == 2 # fn NOT called again - - # Check log has skipped rows - table_path = (node1.pipeline_hash().to_string(), "side_effect_invocations") - records = db.get_all_records(table_path) - df = pl.from_arrow(records) - assert "skipped" in df["status"].to_list() - - def test_t7_no_track_completion_reruns_delivery(self): - """T7: track_completion=False always re-delivers.""" - import polars as pl - from orcapod.side_effects import SideEffectPod, SideEffectJobNode, SideEffectPodConfig - - calls = [] - def fn(data, ctx): - calls.append(True) - - cfg = SideEffectPodConfig(track_completion=False) - pod = SideEffectPod(fn, config=cfg) - stream = _make_stream(2) - db = _make_in_memory_db() - - node1 = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=db) - node1.execute(stream) - - node2 = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=db) - node2.execute(stream) - - assert len(calls) == 4 # called twice per run - - table_path = (node1.pipeline_hash().to_string(), "side_effect_invocations") - records = db.get_all_records(table_path) - df = pl.from_arrow(records) - assert len(df) == 4 # two rows × two runs - assert all(df["status"] == "success") - - def test_t11_invocation_hash_determinism(self): - """T11: Identical inputs produce identical invocation_hash.""" - from orcapod.side_effects import SideEffectPod, SideEffectJobNode, InvocationContext - - ctx_list: list[InvocationContext] = [] - def fn(data, ctx): - ctx_list.append(ctx) - - pod = SideEffectPod(fn) - stream = _make_stream(1) - - node1 = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=_make_in_memory_db()) - node1.execute(stream) - hash1 = ctx_list[0].invocation_hash - ctx_list.clear() - - node2 = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=_make_in_memory_db()) - node2.execute(stream) - hash2 = ctx_list[0].invocation_hash - - assert hash1 == hash2 - - def test_t12_format_id_base64_override(self): - """T12: format_id with base64 encoding returns valid compound.""" - from orcapod.side_effects import ( - SideEffectPod, SideEffectJobNode, InvocationHashConfig, InvocationContext - ) - - ctx_list: list[InvocationContext] = [] - def fn(data, ctx): - ctx_list.append(ctx) - - pod = SideEffectPod(fn) - stream = _make_stream(1) - node = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=_make_in_memory_db()) - node.execute(stream) - - ctx = ctx_list[0] - override = InvocationHashConfig(encoding="base64", component_length=8) - fid = ctx.format_id(override) - - assert fid.startswith("orcapod-") - # Two base64-encoded components of 8 bytes each (11 chars each in base64) - parts = fid[len("orcapod-"):].split("::") - assert len(parts) == 2 - import base64 - for part in parts: - decoded = base64.b64decode(part) - assert len(decoded) == 8 -``` - -- [ ] **Step 2: Run to verify failure** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectJobNodeSync -v -``` -Expected: FAIL (`ImportError: cannot import name 'SideEffectJobNode'`) - -- [ ] **Step 3: Implement `SideEffectJobNode` in `side_effects.py`** - -Append to `src/orcapod/side_effects.py`: - -```python -# --------------------------------------------------------------------------- -# SideEffectJobNode -# --------------------------------------------------------------------------- - - -class SideEffectJobNode(StreamBase): - """DB-backed execution node for side-effect pods. - - Created at pipeline compile time by ``PipelineJob``. Receives a - ``pipeline_database`` via ``attach_databases()``. ``run_id`` is passed - as a call-time keyword argument from the orchestrator. - - Inherits from ``StreamBase`` for identity infrastructure and to satisfy - the ``producer`` / ``upstreams`` / ``output_schema`` contract required - by ``SyncPipelineOrchestrator._materialize_as_stream``. - - Args: - side_effect_pod: The ``SideEffectPod`` this node wraps. - input_stream: The upstream stream at compile time. - label: Optional display label. - """ - - node_type = "side_effect" - - def __init__( - self, - side_effect_pod: SideEffectPod, - input_stream: StreamProtocol, - label: str | None = None, - ) -> None: - self._pod = side_effect_pod - self._input_stream = input_stream - super().__init__(label=label) - self._pipeline_database: ArrowDatabaseProtocol | None = None - self._table_path: tuple[str, ...] | None = None - - # ------------------------------------------------------------------ - # StreamBase interface - # ------------------------------------------------------------------ - - @property - def producer(self): # type: ignore[override] - return self._pod - - @property - def upstreams(self) -> tuple[StreamProtocol, ...]: - return (self._input_stream,) - - def identity_structure(self) -> Any: - return (self._pod, self._pod.argument_symmetry((self._input_stream,))) - - def pipeline_identity_structure(self) -> Any: - return self.identity_structure() - - def computed_label(self) -> str | None: - return self._pod.label - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - return self._input_stream.output_schema(columns=columns, all_info=all_info) - - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - return self._input_stream.keys(columns=columns, all_info=all_info) - - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Standalone iteration — executes delivery with no DB logging.""" - for tag, data in self._input_stream.iter_data(): - result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - pod_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=None, - arrow_hasher=self._pod.data_context.arrow_hasher, - pipeline_database=None, - table_path=None, - ) - if result is not None: - yield result - - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table: - """Collect all rows from ``iter_data()`` into an Arrow table.""" - from orcapod.utils import arrow_utils - - tag_tables = [] - data_tables = [] - for tag, data in self.iter_data(): - tag_tables.append(tag.as_table(columns={"system_tags": True})) - data_tables.append(data.as_table(columns={"source": True})) - if not tag_tables: - return pa.table({}) - return arrow_utils.hstack_tables( - pa.concat_tables(tag_tables), - pa.concat_tables(data_tables), - ) - - # ------------------------------------------------------------------ - # DB attachment - # ------------------------------------------------------------------ - - def attach_databases( - self, - pipeline_database: ArrowDatabaseProtocol | None = None, - ) -> None: - """Attach or detach the pipeline database. - - Called by ``PipelineJob._distribute_databases()``. The table path is - scoped to ``(self.pipeline_hash().to_string(), "side_effect_invocations")``. - - Args: - pipeline_database: Pre-scoped pipeline DB (at pipeline root), - or ``None`` to detach. - """ - self._pipeline_database = pipeline_database - if pipeline_database is not None: - self._table_path = ( - self.pipeline_hash().to_string(), - "side_effect_invocations", - ) - else: - self._table_path = None - - # ------------------------------------------------------------------ - # Sync execution - # ------------------------------------------------------------------ - - def execute( - self, - input_stream: StreamProtocol, - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, - ) -> list[tuple[TagProtocol, DataProtocol]]: - """Execute side-effect delivery for all rows in ``input_stream``. - - Args: - input_stream: Stream of ``(tag, data)`` pairs to process. - observer: Optional execution observer (currently unused). - run_id: Pipeline run identifier forwarded from the orchestrator. - - Returns: - List of ``(tag, data)`` tuples — the pass-through rows. - """ - results = [] - for tag, data in input_stream.iter_data(): - result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - pod_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=run_id, - arrow_hasher=self._pod.data_context.arrow_hasher, - pipeline_database=self._pipeline_database, - table_path=self._table_path, - ) - if result is not None: - results.append(result) - return results -``` - -- [ ] **Step 4: Run DB tests** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectJobNodeSync -v -``` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/side_effects.py -git commit -m "feat(side-effects): add SideEffectJobNode with sync execute and DB logging" -``` - ---- - -## Task 8: `SideEffectJobNode.async_execute()` - -**Files:** -- Modify: `src/orcapod/side_effects.py` -- Test: `tests/test_core/side_effect_pod/test_side_effect_pod.py` - -- [ ] **Step 1: Write failing async tests T13–T14** - -Add to `tests/test_core/side_effect_pod/test_side_effect_pod.py`: - -```python -# --------------------------------------------------------------------------- -# Task 8 tests — async execution -# --------------------------------------------------------------------------- - - -class TestSideEffectJobNodeAsync: - """T13–T14: async_execute via channels.""" - - def test_t13_async_execute_basic(self): - """T13: async_execute processes all rows and writes log.""" - import asyncio - import polars as pl - from orcapod.side_effects import SideEffectPod, SideEffectJobNode - from orcapod.channels import Channel - - calls = [] - def fn(data, ctx): - calls.append(True) - - pod = SideEffectPod(fn) - stream = _make_stream(3) - db = _make_in_memory_db() - node = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=db) - - async def _run(): - ch_in = Channel(buffer_size=10) - ch_out = Channel(buffer_size=10) - - async def feed(): - for tag, data in stream.iter_data(): - await ch_in.writer.send((tag, data)) - await ch_in.writer.close() - - await asyncio.gather( - feed(), - node.async_execute( - [ch_in.reader], ch_out.writer, run_id="test-run-1" - ), - ) - return await ch_out.reader.collect() - - results = asyncio.run(_run()) - assert len(results) == 3 - assert len(calls) == 3 - - table_path = (node.pipeline_hash().to_string(), "side_effect_invocations") - records = db.get_all_records(table_path) - df = pl.from_arrow(records) - assert len(df) == 3 - assert all(df["status"] == "success") - - def test_t14_async_execute_parallel(self): - """T14: Concurrent delivery via max_concurrency — all rows complete.""" - import asyncio - from orcapod.side_effects import SideEffectPod, SideEffectJobNode - from orcapod.channels import Channel - - results_store = [] - def fn(data, ctx): - results_store.append(True) - - pod = SideEffectPod(fn) - stream = _make_stream(10) - db = _make_in_memory_db() - node = SideEffectJobNode(side_effect_pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=db) - - async def _run(): - ch_in = Channel(buffer_size=20) - ch_out = Channel(buffer_size=20) - - async def feed(): - for tag, data in stream.iter_data(): - await ch_in.writer.send((tag, data)) - await ch_in.writer.close() - - await asyncio.gather( - feed(), - node.async_execute([ch_in.reader], ch_out.writer), - ) - return await ch_out.reader.collect() - - out = asyncio.run(_run()) - assert len(out) == 10 - assert len(results_store) == 10 -``` - -- [ ] **Step 2: Run to verify failure** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectJobNodeAsync -v -``` -Expected: FAIL (`AttributeError: 'SideEffectJobNode' has no attribute 'async_execute'`) - -- [ ] **Step 3: Implement `async_execute` on `SideEffectJobNode`** - -Inside the `SideEffectJobNode` class in `src/orcapod/side_effects.py`, add after `execute()`: - -```python - async def async_execute( - self, - inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, - ) -> None: - """Async side-effect delivery with per-row concurrency control. - - Reads from ``inputs[0]``, dispatches each row as an independent - async task via ``asyncio.TaskGroup``. Emits non-``None`` results to - ``output``. Always closes ``output`` in a ``finally`` block. - - Args: - inputs: Single-element sequence containing the input channel. - output: Writable channel for pass-through ``(tag, data)`` pairs. - observer: Optional execution observer (currently unused). - run_id: Pipeline run identifier from the orchestrator. - """ - try: - async def process_one(tag: TagProtocol, data: DataProtocol) -> None: - result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - pod_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=run_id, - arrow_hasher=self._pod.data_context.arrow_hasher, - pipeline_database=self._pipeline_database, - table_path=self._table_path, - ) - if result is not None: - await output.send(result) - - async with asyncio.TaskGroup() as tg: - async for tag, data in inputs[0]: - tg.create_task(process_one(tag, data)) - finally: - await output.close() -``` - -- [ ] **Step 4: Run async tests** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectJobNodeAsync -v -``` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/side_effects.py -git commit -m "feat(side-effects): add SideEffectJobNode.async_execute" -``` - ---- - -## Task 9: Pipeline integration + orchestrators - -**Files:** -- Modify: `src/orcapod/pipeline/base.py` -- Modify: `src/orcapod/pipeline/job.py` -- Modify: `src/orcapod/pipeline/sync_orchestrator.py` -- Modify: `src/orcapod/pipeline/async_orchestrator.py` -- Test: `tests/test_core/side_effect_pod/test_side_effect_pod.py` - -- [ ] **Step 1: Write failing pipeline test T15** - -Add to `tests/test_core/side_effect_pod/test_side_effect_pod.py`: - -```python -# --------------------------------------------------------------------------- -# Task 9 tests — full pipeline integration -# --------------------------------------------------------------------------- - - -class TestSideEffectPodPipelineIntegration: - """T15: Side-effect pod inside a PipelineJob.""" - - def test_t15_pipeline_composition_mid_pipeline(self): - """T15: Pod runs mid-pipeline; downstream node receives filtered output.""" - import polars as pl - from orcapod.pipeline import PipelineJob - from orcapod.core.sources import DictSource - from orcapod.side_effects import SideEffectPod, SideEffectPodConfig - from orcapod.databases.in_memory import InMemoryDatabase - - delivery_log = [] - - @SideEffectPod - def log_delivery(data, ctx): - delivery_log.append(dict(data)) - - db = InMemoryDatabase() - - with PipelineJob(name="test_pipeline", store=db) as job: - source = DictSource( - [{"id": 0, "value": 10}, {"id": 1, "value": 20}], - tag_columns=["id"], - ) - stream = source.stream() - side_effect_stream = log_delivery.process(stream) - - job.run() - - # Delivery log was populated - assert len(delivery_log) == 2 - - # Invocation log written to DB - from orcapod.pipeline.base import AbstractPipelineBase - # Find the side_effect node's pipeline hash - side_effect_nodes = [ - n for n in job._persistent_node_map.values() - if n.node_type == "side_effect" - ] - assert len(side_effect_nodes) == 1 - node = side_effect_nodes[0] - table_path = (node.pipeline_hash().to_string(), "side_effect_invocations") - records = db.at(*job.name).get_all_records(table_path) - assert records is not None - df = pl.from_arrow(records) - assert len(df) == 2 - assert all(df["status"] == "success") -``` - -- [ ] **Step 2: Run to verify failure** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectPodPipelineIntegration -v -``` -Expected: FAIL (TypeError: Unknown node type `"side_effect"`) - -- [ ] **Step 3: Add `side_effect_node_class` abstract property to `AbstractPipelineBase`** - -Open `src/orcapod/pipeline/base.py`. After `operator_node_class` abstract property, add: - -```python - @property - @abstractmethod - def side_effect_node_class(self) -> type: - """Node class to use for side-effect pod invocations — e.g. ``SideEffectJobNode``.""" - ... -``` - -- [ ] **Step 4: Add `SideEffectInvocation` branch to `AbstractPipelineBase.compile()`** - -In `src/orcapod/pipeline/base.py`, in the `compile()` method, change the final `else:` branch of the `isinstance` dispatch. Currently it reads: - -```python - if isinstance(inv, FunctionInvocation): - node_map[key] = self.function_node_class( - function_pod=inv.pod, - input_stream=upstream_nodes[0], - label=inv.label, - ) - else: - node_map[key] = self.operator_node_class( - operator=inv.pod, - input_streams=tuple(upstream_nodes), - label=inv.label, - ) -``` - -Change to: - -```python - if isinstance(inv, FunctionInvocation): - node_map[key] = self.function_node_class( - function_pod=inv.pod, - input_stream=upstream_nodes[0], - label=inv.label, - ) - elif isinstance(inv, SideEffectInvocation): - node_map[key] = self.side_effect_node_class( - side_effect_pod=inv.pod, - input_stream=upstream_nodes[0], - label=inv.label, - ) - else: - node_map[key] = self.operator_node_class( - operator=inv.pod, - input_streams=tuple(upstream_nodes), - label=inv.label, - ) -``` - -- [ ] **Step 5: Add `side_effect_node_class` and `_distribute_databases` branch to `PipelineJob`** - -Open `src/orcapod/pipeline/job.py`. - -Add import at the top: -```python -from orcapod.side_effects import SideEffectJobNode -``` - -After `operator_node_class = OperatorJobNode`, add: -```python - side_effect_node_class = SideEffectJobNode -``` - -In `_distribute_databases()`, after the `elif isinstance(node, OperatorJobNode):` block, add: -```python - elif isinstance(node, SideEffectJobNode): - node.attach_databases(pipeline_database=pipeline_db) -``` - -- [ ] **Step 6: Add `elif is_side_effect_node` branch to sync orchestrator** - -Open `src/orcapod/pipeline/sync_orchestrator.py`. - -Add import: -```python -from orcapod.protocols.node_protocols import ( - is_function_node, - is_operator_node, - is_side_effect_node, # add - is_source_node, -) -``` - -In `SyncPipelineOrchestrator.run()`, after the `elif is_operator_node(node):` block and before the `else:` raise, add: - -```python - elif is_side_effect_node(node): - upstream_buf = self._gather_upstream(node, graph, buffers) - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream(upstream_buf, upstream_node) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) -``` - -- [ ] **Step 7: Add `elif is_side_effect_node` branch to async orchestrator** - -Open `src/orcapod/pipeline/async_orchestrator.py`. - -Add import: -```python -from orcapod.protocols.node_protocols import ( - is_function_node, - is_operator_node, - is_side_effect_node, # add - is_source_node, -) -``` - -In `_run_async()`, after the `elif is_operator_node(node):` block and before the `else:` raise, add: - -```python - elif is_side_effect_node(node): - predecessors = in_edges.get(node, []) - if not predecessors: - raise ValueError( - f"SideEffectNode expects exactly 1 upstream, got 0" - ) - input_reader = edge_readers[(predecessors[0], node)] - tg.create_task( - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ) - ) -``` - -- [ ] **Step 8: Run pipeline integration test** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py::TestSideEffectPodPipelineIntegration -v -``` -Expected: PASS - -- [ ] **Step 9: Run full test suite to check for regressions** - -```bash -uv run pytest tests/ -x -q -``` -Expected: All existing tests pass, new tests pass. - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/pipeline/base.py src/orcapod/pipeline/job.py src/orcapod/pipeline/sync_orchestrator.py src/orcapod/pipeline/async_orchestrator.py -git commit -m "feat(pipeline): wire SideEffectJobNode into compile(), _distribute_databases(), and orchestrators" -``` - ---- - -## Task 10: `__init__.py` re-exports - -**Files:** -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Add re-exports** - -Open `src/orcapod/__init__.py`. Add after the `function_pod` import block: - -```python -from .side_effects import ( - InvocationContext, - InvocationHashConfig, - SideEffectPod, - SideEffectPodConfig, - side_effect_pod, - sink_pod, - tap_pod, -) -``` - -Add these names to `__all__`: -```python - "InvocationContext", - "InvocationHashConfig", - "SideEffectPod", - "SideEffectPodConfig", - "side_effect_pod", - "sink_pod", - "tap_pod", -``` - -- [ ] **Step 2: Verify re-exports work** - -```bash -uv run python -c " -from orcapod import ( - InvocationContext, InvocationHashConfig, SideEffectPod, - SideEffectPodConfig, side_effect_pod, sink_pod, tap_pod, -) -print('all re-exports ok') -" -``` -Expected: `all re-exports ok` - -- [ ] **Step 3: Run complete test suite one final time** - -```bash -uv run pytest tests/ -q -``` -Expected: All tests pass. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/__init__.py -git commit -m "feat(orcapod): re-export SideEffectPod public API from orcapod.__init__" -``` - ---- - -## Self-Review - -### Spec coverage - -| Spec section | Covered by task | -|---|---| -| `InvocationHashConfig` | Task 1 | -| `SideEffectPodConfig` | Task 1 | -| `InvocationContext` + `format_id()` | Task 1 | -| `SideEffectPodProtocol` | Task 2 | -| `SideEffectInvocation` | Task 3 | -| Tracker chain | Task 4 | -| `SideEffectPod` + `SideEffectPodStream` | Task 5 | -| Decorators `sink_pod`, `tap_pod`, `side_effect_pod` | Task 5 | -| `SideEffectNodeProtocol` + `is_side_effect_node()` | Task 6 | -| `SideEffectJobNode` sync + DB writes | Task 7 | -| Invocation table schema (no `invocation_hash` column) | Task 7 | -| `SideEffectJobNode.async_execute()` | Task 8 | -| `AbstractPipelineBase.compile()` branch | Task 9 | -| `PipelineJob._distribute_databases()` branch | Task 9 | -| Sync orchestrator dispatch | Task 9 | -| Async orchestrator dispatch | Task 9 | -| `__init__.py` re-exports | Task 10 | -| T1–T18 test scenarios | Tasks 5, 7, 8, 9 | -| `docs/concepts/side-effect-pods.md` | ⚠️ Not covered — doc stub is out-of-scope for implementation; add as a follow-up task if required | - -### Placeholder scan - -No placeholders found — all steps contain complete code. - -### Type consistency - -- `SideEffectJobNode.execute()` takes `input_stream: StreamProtocol` → matches `SideEffectNodeProtocol.execute()` signature ✓ -- `SideEffectJobNode.async_execute()` takes `inputs: Sequence[ReadableChannel[...]]` → matches `SideEffectNodeProtocol.async_execute()` signature ✓ -- `_execute_side_effect_row()` referenced in both `SideEffectPodStream.iter_data()` and `SideEffectJobNode.execute()` → same function ✓ -- `_write_invocation_row()` uses `pa.large_string()` for all string columns → matches invocation table DDL in spec ✓ -- `is_side_effect_node(node)` checks `node.node_type == "side_effect"` → `SideEffectJobNode.node_type = "side_effect"` ✓ -- `SideEffectInvocation` imported in `pipeline/base.py` from `pod_invocation.py` → added in Task 3 ✓ -- `side_effect_node_class` abstract on `AbstractPipelineBase` → concrete on `PipelineJob` ✓ - ---- - -## Execution Options - -Plan complete and saved to `superpowers/plans/2026-07-14-itl-525-side-effect-pod.md`. - -**1. Subagent-Driven (recommended)** — Fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/superpowers/plans/2026-07-16-empty-data-ephemeral-propagation.md b/superpowers/plans/2026-07-16-empty-data-ephemeral-propagation.md deleted file mode 100644 index 1ddab7e6b..000000000 --- a/superpowers/plans/2026-07-16-empty-data-ephemeral-propagation.md +++ /dev/null @@ -1,1589 +0,0 @@ -# EmptyData + Ephemeral Result Propagation Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Propagate ephemeral result misses as `EmptyData` tokens through the pipeline, enabling downstream pods to recover via their own result cache rather than failing silently or blocking. - -**Architecture:** Add `EmptyData(Data)` — a payload-free `Data` subclass carrying an optional cached content hash — then wire it into the pipeline DB read/write path and the downstream execution guard. `EmptyData.content_hash()` returns the stored hash, making `ResultCache.lookup()` work transparently; a guard in `_process_data_internal()` raises `EphemeralResultMissingError` on downstream cache miss rather than attempting computation. - -**Tech Stack:** Python, PyArrow, Polars (for anti-join in `_fetch_joined_records`), `orcapod` internal APIs (`ResultCache`, `CachedFunctionPod`, `FunctionJobNode`, `ArrowTableStream`). - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/errors.py` | Modify | Add `EmptyDataAccessError`, `EmptyDataHashMissingError`, `EphemeralResultMissingError` | -| `src/orcapod/core/datagrams/tag_data.py` | Modify | Add `EmptyData(Data)` class | -| `src/orcapod/core/datagrams/__init__.py` | Modify | Export `EmptyData` | -| `src/orcapod/core/cached_function_pod.py` | Modify | Expose `result_cache` property | -| `src/orcapod/core/nodes/function_node.py` | Modify | Write path, read path, downstream guard | -| `tests/test_core/test_empty_data.py` | Create | Unit tests for `EmptyData` | -| `tests/test_core/test_result_cache.py` | Modify | Add lookup-via-`EmptyData` tests | -| `tests/test_core/nodes/test_function_node_empty_data.py` | Create | Integration tests for ephemeral-miss flow | - ---- - -## Task 1: Exception types - -**Files:** -- Modify: `src/orcapod/errors.py` -- Test: `tests/test_core/test_empty_data.py` (created here, extended in Task 2) - -- [ ] **Step 1: Create the test file with exception smoke-tests** - -```python -# tests/test_core/test_empty_data.py -"""Tests for EmptyData and its associated exception types.""" -from __future__ import annotations - -import pytest - -from orcapod.errors import ( - EmptyDataAccessError, - EmptyDataHashMissingError, - EphemeralResultMissingError, -) - - -class TestExceptionTypes: - def test_empty_data_access_error_is_exception(self): - exc = EmptyDataAccessError("sentinel", "as_dict") - assert isinstance(exc, Exception) - assert exc.empty_data == "sentinel" - assert exc.method_name == "as_dict" - - def test_empty_data_hash_missing_error_is_exception(self): - exc = EmptyDataHashMissingError("sentinel") - assert isinstance(exc, Exception) - assert exc.empty_data == "sentinel" - - def test_ephemeral_result_missing_error_is_exception(self): - exc = EphemeralResultMissingError( - tag="tag", - cached_content_hash=None, - node_identity_path=("a", "b"), - message="gone", - ) - assert isinstance(exc, Exception) - assert exc.tag == "tag" - assert exc.cached_content_hash is None - assert exc.node_identity_path == ("a", "b") - assert "gone" in str(exc) -``` - -- [ ] **Step 2: Run the test to verify it fails** - -``` -uv run pytest tests/test_core/test_empty_data.py::TestExceptionTypes -v -``` - -Expected: `ImportError` — `EmptyDataAccessError` not found in `errors.py`. - -- [ ] **Step 3: Add exception classes to `src/orcapod/errors.py`** - -Append at the end of the file: - -```python -class EmptyDataAccessError(Exception): - """Raised when a payload-access method is called on an ``EmptyData`` instance. - - ``EmptyData`` carries no data payload. Any method that would access the - underlying columns (``as_dict``, ``as_table``, ``keys``, ``schema``, - ``arrow_schema``, ``identity_structure``) raises this exception instead of - returning empty or ``None`` results. - - Attributes: - empty_data: The ``EmptyData`` instance on which the method was called. - method_name: The name of the method that was called. - """ - - def __init__(self, empty_data: object, method_name: str) -> None: - self.empty_data = empty_data - self.method_name = method_name - super().__init__( - f"Cannot call {method_name!r} on EmptyData — " - "this instance carries no data payload. " - "Check isinstance(data, EmptyData) before accessing the payload." - ) - - -class EmptyDataHashMissingError(Exception): - """Raised when ``content_hash()`` is called on an ``EmptyData`` that has no cached hash. - - An ``EmptyData`` constructed without a ``cached_content_hash`` is in degraded - mode (old pipeline DB row lacking the hash column). Any code path that tries - to use the hash fails loudly here. - - Attributes: - empty_data: The ``EmptyData`` instance on which ``content_hash()`` was called. - """ - - def __init__(self, empty_data: object) -> None: - self.empty_data = empty_data - super().__init__( - "content_hash() called on EmptyData with no cached_content_hash. " - "This EmptyData was constructed from a pipeline DB row that predates " - "the INPUT_DATA_HASH_COL column. Flow-through is unavailable for this row." - ) - - -class EphemeralResultMissingError(Exception): - """Raised when a downstream pod has a cache miss for an ``EmptyData`` input. - - Indicates that the upstream ephemeral result is gone AND the downstream pod - has not yet computed a result for the same input content hash. No recovery - is possible — the information has been lost. - - Attributes: - tag: The tag associated with the missing input. - cached_content_hash: The cached content hash from the ``EmptyData``, or - ``None`` if the ``EmptyData`` itself lacked a hash. - node_identity_path: Identity path of the downstream node raising this error. - """ - - def __init__( - self, - tag: object, - cached_content_hash: object, - node_identity_path: tuple[str, ...], - message: str, - ) -> None: - self.tag = tag - self.cached_content_hash = cached_content_hash - self.node_identity_path = node_identity_path - super().__init__( - f"{message} " - f"node={node_identity_path!r} " - f"cached_content_hash={cached_content_hash!r}" - ) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -``` -uv run pytest tests/test_core/test_empty_data.py::TestExceptionTypes -v -``` - -Expected: PASS (3 tests). - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/errors.py tests/test_core/test_empty_data.py -git commit -m "feat(errors): add EmptyDataAccessError, EmptyDataHashMissingError, EphemeralResultMissingError (ITL-534)" -``` - ---- - -## Task 2: `EmptyData` class - -**Files:** -- Modify: `src/orcapod/core/datagrams/tag_data.py` -- Test: `tests/test_core/test_empty_data.py` (extend) - -- [ ] **Step 1: Add failing tests for `EmptyData`** - -Append to `tests/test_core/test_empty_data.py`: - -```python -import uuid - -import pyarrow as pa - -from orcapod.core.datagrams import Data -from orcapod.core.datagrams.tag_data import EmptyData -from orcapod.types import ContentHash - - -def _make_hash(hex_str: str = "a" * 64) -> ContentHash: - return ContentHash("arrow_v2.1", bytes.fromhex(hex_str)) - - -class TestEmptyDataSubclass: - def test_is_data_subclass(self): - assert issubclass(EmptyData, Data) - - def test_instance_is_data(self): - ed = EmptyData() - assert isinstance(ed, Data) - - -class TestEmptyDataContentHash: - def test_returns_cached_hash_when_set(self): - h = _make_hash() - ed = EmptyData(cached_content_hash=h) - assert ed.content_hash() == h - - def test_raises_when_no_cached_hash(self): - ed = EmptyData() - with pytest.raises(EmptyDataHashMissingError): - ed.content_hash() - - def test_cached_content_hash_property(self): - h = _make_hash() - ed = EmptyData(cached_content_hash=h) - assert ed.cached_content_hash is h - - def test_cached_content_hash_property_none(self): - ed = EmptyData() - assert ed.cached_content_hash is None - - -class TestEmptyDataPayloadAccess: - """All payload-access methods must raise EmptyDataAccessError.""" - - def setup_method(self): - self.ed = EmptyData(cached_content_hash=_make_hash()) - - def test_as_dict_raises(self): - with pytest.raises(EmptyDataAccessError): - self.ed.as_dict() - - def test_as_table_raises(self): - with pytest.raises(EmptyDataAccessError): - self.ed.as_table() - - def test_keys_raises(self): - with pytest.raises(EmptyDataAccessError): - self.ed.keys() - - def test_schema_raises(self): - with pytest.raises(EmptyDataAccessError): - self.ed.schema() - - def test_arrow_schema_raises(self): - with pytest.raises(EmptyDataAccessError): - self.ed.arrow_schema() - - def test_identity_structure_raises(self): - with pytest.raises(EmptyDataAccessError): - self.ed.identity_structure() - - -class TestEmptyDataSourceInfo: - def test_empty_source_info_none_by_default(self): - ed = EmptyData() - assert ed.empty_source_info is None - - def test_empty_source_info_stored(self): - si = {"source_id": "abc", "record_id": None} - ed = EmptyData(empty_source_info=si) - assert ed.empty_source_info == si - - -class TestEmptyDataMetadata: - def test_record_uuid_assigned(self): - ed = EmptyData() - assert ed.datagram_uuid is not None - - def test_custom_record_uuid(self): - uid = uuid.uuid4() - ed = EmptyData(record_uuid=uid) - assert ed.datagram_uuid == uid -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -``` -uv run pytest tests/test_core/test_empty_data.py -k "not TestExceptionTypes" -v -``` - -Expected: `ImportError` — `EmptyData` not found. - -- [ ] **Step 3: Implement `EmptyData` in `tag_data.py`** - -Add after the `Data` class (at the bottom of `src/orcapod/core/datagrams/tag_data.py`): - -```python -# --------------------------------------------------------------------------- -# EmptyData -# --------------------------------------------------------------------------- - - -class EmptyData(Data): - """A ``Data``-shaped token representing missing data. - - ``EmptyData`` is produced when an upstream pod's ephemeral result has - expired or been pruned. It carries all normal datagram metadata (data - context, record UUID) but has no data payload. Every payload-access - method raises ``EmptyDataAccessError`` — callers must - ``isinstance(data, EmptyData)`` before touching columns. - - ``content_hash()`` is overridden to return ``cached_content_hash`` (the - hash of the original input data this token stands in for). This makes - ``ResultCache.lookup(empty_data)`` work transparently without changes to - the cache infrastructure. If ``cached_content_hash`` is ``None`` (old - pipeline DB row lacking the hash column), ``content_hash()`` raises - ``EmptyDataHashMissingError`` loudly. - - ``empty_source_info`` is an optional provenance field for the future - tag-row reconstruction follow-up. This PR defines the field; the write - logic is deferred. - - Args: - cached_content_hash: The content hash of the original data this token - represents. ``None`` for old-format rows lacking the hash column. - empty_source_info: Optional provenance dict for tag-row reconstruction - (follow-up). Keys match tag-row source columns; ``record_id`` may - be ``None``. - python_schema: Optional schema hint (passed to parent). - data_context: Data context key or instance (passed to parent). - record_uuid: Optional explicit UUID for this token. - """ - - def __init__( - self, - cached_content_hash: "ContentHash | None" = None, - empty_source_info: "dict[str, str | None] | None" = None, - python_schema: "SchemaLike | None" = None, - data_context: "str | contexts.DataContext | None" = None, - record_uuid: "uuid.UUID | None" = None, - ) -> None: - # Initialise the parent with an empty dict — no payload columns. - super().__init__( - {}, - python_schema=python_schema, - data_context=data_context, - record_uuid=record_uuid, - ) - self._cached_content_hash = cached_content_hash - self._empty_source_info = empty_source_info - - # ------------------------------------------------------------------ - # Content identity - # ------------------------------------------------------------------ - - def content_hash(self, hasher=None) -> "ContentHash": - """Return the cached content hash or raise ``EmptyDataHashMissingError``. - - The returned hash is the hash of the original input data this token - represents, enabling ``ResultCache.lookup(empty_data)`` to find the - downstream cached result without accessing the missing payload. - - Args: - hasher: Ignored — ``EmptyData`` uses the stored hash directly. - - Returns: - The ``cached_content_hash`` set at construction. - - Raises: - EmptyDataHashMissingError: If ``cached_content_hash`` is ``None``. - """ - from orcapod.errors import EmptyDataHashMissingError - - if self._cached_content_hash is None: - raise EmptyDataHashMissingError(self) - return self._cached_content_hash - - def identity_structure(self) -> "Any": - """Always raises ``EmptyDataAccessError`` — no payload to hash.""" - from orcapod.errors import EmptyDataAccessError - - raise EmptyDataAccessError(self, "identity_structure") - - # ------------------------------------------------------------------ - # Payload-access overrides — all raise EmptyDataAccessError - # ------------------------------------------------------------------ - - def as_dict(self, *, columns=None, all_info: bool = False) -> "dict": - from orcapod.errors import EmptyDataAccessError - - raise EmptyDataAccessError(self, "as_dict") - - def as_table(self, *, columns=None, all_info: bool = False) -> "pa.Table": - from orcapod.errors import EmptyDataAccessError - - raise EmptyDataAccessError(self, "as_table") - - def keys(self, *, columns=None, all_info: bool = False) -> "tuple[str, ...]": - from orcapod.errors import EmptyDataAccessError - - raise EmptyDataAccessError(self, "keys") - - def schema(self, *, columns=None, all_info: bool = False) -> "Schema": - from orcapod.errors import EmptyDataAccessError - - raise EmptyDataAccessError(self, "schema") - - def arrow_schema(self, *, columns=None, all_info: bool = False) -> "pa.Schema": - from orcapod.errors import EmptyDataAccessError - - raise EmptyDataAccessError(self, "arrow_schema") - - # ------------------------------------------------------------------ - # Read-only accessors - # ------------------------------------------------------------------ - - @property - def cached_content_hash(self) -> "ContentHash | None": - """The stored content hash, or ``None`` if absent (old-format row).""" - return self._cached_content_hash - - @property - def empty_source_info(self) -> "dict[str, str | None] | None": - """Optional provenance dict for future tag-row reconstruction.""" - return self._empty_source_info -``` - -The import block at the top of `tag_data.py` already includes `Any`, `Schema`, `SchemaLike`, `contexts`, `uuid`, and `pa` (lazy). No new imports needed. - -- [ ] **Step 4: Run the tests** - -``` -uv run pytest tests/test_core/test_empty_data.py -v -``` - -Expected: All tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/core/datagrams/tag_data.py tests/test_core/test_empty_data.py -git commit -m "feat(datagrams): add EmptyData subclass of Data (ITL-534)" -``` - ---- - -## Task 3: Export `EmptyData` - -**Files:** -- Modify: `src/orcapod/core/datagrams/__init__.py` - -- [ ] **Step 1: Add `EmptyData` to the datagrams package exports** - -Edit `src/orcapod/core/datagrams/__init__.py`: - -```python -from .datagram import Datagram -from .tag_data import Data, EmptyData, Tag - -__all__ = [ - "Datagram", - "Tag", - "Data", - "EmptyData", -] -``` - -- [ ] **Step 2: Verify import works** - -``` -uv run python -c "from orcapod.core.datagrams import EmptyData; print('ok')" -``` - -Expected: `ok` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/core/datagrams/__init__.py -git commit -m "feat(datagrams): export EmptyData from datagrams package (ITL-534)" -``` - ---- - -## Task 4: Expose `result_cache` on `CachedFunctionPod` - -**Files:** -- Modify: `src/orcapod/core/cached_function_pod.py` -- Test: `tests/test_core/test_empty_data.py` (extend with ResultCache lookup test) - -`_process_data_internal` (Task 8) will need to call `result_cache.lookup(empty_data)` directly, bypassing `process_data()` to avoid triggering computation. Expose `_cache` as a public `result_cache` property. - -- [ ] **Step 1: Add a failing test** - -Append to `tests/test_core/test_empty_data.py`: - -```python -from orcapod.core.cached_function_pod import CachedFunctionPod -from orcapod.core.result_cache import ResultCache - - -class TestCachedFunctionPodResultCacheProperty: - def test_result_cache_is_result_cache_instance(self): - """CachedFunctionPod exposes result_cache as a ResultCache.""" - import pyarrow as pa - from orcapod.core.data_function import PythonDataFunction - from orcapod.core.function_pod import FunctionPod - from orcapod.databases import InMemoryArrowDatabase - - pf = PythonDataFunction(lambda x: x * 2, output_keys="result") - pod = FunctionPod(pf) - cached_pod = CachedFunctionPod(pod, result_database=InMemoryArrowDatabase()) - assert isinstance(cached_pod.result_cache, ResultCache) -``` - -- [ ] **Step 2: Run to verify it fails** - -``` -uv run pytest tests/test_core/test_empty_data.py::TestCachedFunctionPodResultCacheProperty -v -``` - -Expected: `AttributeError: 'CachedFunctionPod' object has no attribute 'result_cache'`. - -- [ ] **Step 3: Add `result_cache` property to `CachedFunctionPod`** - -In `src/orcapod/core/cached_function_pod.py`, add after the `record_path` property: - -```python - @property - def result_cache(self) -> ResultCache: - """The underlying ``ResultCache`` instance. - - Exposed so that callers (e.g. ``FunctionJobNode._process_data_internal``) - can perform a cache-only lookup without triggering computation — necessary - when the input is an ``EmptyData`` token. - """ - return self._cache -``` - -- [ ] **Step 4: Run the test to verify it passes** - -``` -uv run pytest tests/test_core/test_empty_data.py::TestCachedFunctionPodResultCacheProperty -v -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/core/cached_function_pod.py tests/test_core/test_empty_data.py -git commit -m "feat(cached_function_pod): expose result_cache property (ITL-534)" -``` - ---- - -## Task 5: Extend `ResultCache` lookup with `EmptyData` - -**Files:** -- Test: `tests/test_core/test_result_cache.py` - -No implementation change needed — `ResultCache.lookup(empty_data)` already works because `EmptyData.content_hash()` returns the cached hash. This task adds tests to prove it. - -- [ ] **Step 1: Append tests to `tests/test_core/test_result_cache.py`** - -```python -# --- Add at the bottom of tests/test_core/test_result_cache.py --- - -from orcapod.core.datagrams.tag_data import EmptyData -from orcapod.types import ContentHash - - -class TestLookupViaEmptyData: - def test_empty_data_cache_hit(self): - """ResultCache.lookup works for EmptyData with a matching cached hash.""" - cache, _ = _make_cache() - pf = _make_pf() - input_pkt = Data({"x": 10}) - _compute_and_store(cache, pf, input_pkt) - - # EmptyData with the same hash as input_pkt - empty = EmptyData(cached_content_hash=input_pkt.content_hash()) - found = cache.lookup(empty) - assert found is not None - assert found.as_dict()["result"] == 20 - - def test_empty_data_cache_miss(self): - """ResultCache.lookup returns None for EmptyData with an unknown hash.""" - cache, _ = _make_cache() - # Store a result so the DB is non-empty - pf = _make_pf() - _compute_and_store(cache, pf, Data({"x": 10})) - - # EmptyData with a hash that was never stored - unknown_hash = ContentHash("arrow_v2.1", bytes(32)) - empty = EmptyData(cached_content_hash=unknown_hash) - assert cache.lookup(empty) is None -``` - -- [ ] **Step 2: Run to verify they pass immediately** - -``` -uv run pytest tests/test_core/test_result_cache.py::TestLookupViaEmptyData -v -``` - -Expected: PASS (transparent — no implementation change needed). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_core/test_result_cache.py -git commit -m "test(result_cache): add EmptyData cache-lookup tests (ITL-534)" -``` - ---- - -## Task 6: Store `INPUT_DATA_HASH_COL` in the tag table write path - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines 1488–1615, `add_pipeline_record`) -- Test: `tests/test_core/nodes/test_function_node_empty_data.py` (created here) - -- [ ] **Step 1: Create the test file with a failing test for the write path** - -```python -# tests/test_core/nodes/test_function_node_empty_data.py -"""Tests for EmptyData integration in FunctionJobNode.""" -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.datagrams import Data -from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes.function_node import FunctionJobNode -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.system_constants import constants - - -def double_value(value: int) -> int: - return value * 2 - - -@pytest.fixture -def persistent_node(): - """FunctionJobNode with persistent databases, two input rows.""" - table = pa.table({ - "key": pa.array(["a", "b"], type=pa.large_string()), - "value": pa.array([1, 2], type=pa.int64()), - }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - return FunctionJobNode( - pod, src, - pipeline_database=InMemoryArrowDatabase(), - result_database=InMemoryArrowDatabase(), - ) - - -class TestAddPipelineRecordStoresInputHash: - def test_pipeline_record_contains_input_data_hash(self, persistent_node): - """add_pipeline_record now stores INPUT_DATA_HASH_COL in the pipeline DB.""" - node = persistent_node - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - - all_records = node._pipeline_database.get_all_records(node.node_identity_path) - assert all_records is not None - assert constants.INPUT_DATA_HASH_COL in all_records.column_names - - def test_stored_hash_matches_input_content_hash(self, persistent_node): - """The stored INPUT_DATA_HASH_COL value matches input_data.content_hash().""" - node = persistent_node - input_pairs = list(node._input_stream.iter_data()) - tag0, data0 = input_pairs[0] - node.execute_data(tag0, data0) - - all_records = node._pipeline_database.get_all_records(node.node_identity_path) - assert all_records is not None - stored_hashes = all_records.column(constants.INPUT_DATA_HASH_COL).to_pylist() - assert data0.content_hash().to_string() in stored_hashes -``` - -- [ ] **Step 2: Run to verify tests fail** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestAddPipelineRecordStoresInputHash -v -``` - -Expected: FAIL — `INPUT_DATA_HASH_COL` not in column names. - -- [ ] **Step 3: Add `INPUT_DATA_HASH_COL` to `add_pipeline_record`** - -In `src/orcapod/core/nodes/function_node.py`, find `add_pipeline_record` (around line 1488) and locate the `meta_table` construction (around line 1602). Add the new column: - -```python - meta_table = pa.table( - { - constants.DATA_RECORD_ID: pa.array( - [data_record_id.bytes], type=pa.large_binary() - ), - constants.NODE_CONTENT_HASH_COL: pa.array( - [self.content_hash().to_string()], type=pa.large_string() - ), - constants.INPUT_DATA_HASH_COL: pa.array( # NEW - [input_data.content_hash().to_string()], type=pa.large_string() - ), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array( - [input_data.data_context_key], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array( - [computed], type=pa.bool_() - ), - constants.IS_EPHEMERAL_COL: pa.array( - [is_ephemeral], type=pa.bool_() - ), - _PIPELINE_BASE_ENTRY_ID_COL: pa.array( - [base_entry_id], type=pa.large_binary() - ), - _PIPELINE_RECOMPUTATION_INDEX_COL: pa.array( - [new_index], type=pa.int32() - ), - } - ) -``` - -- [ ] **Step 4: Run tests** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestAddPipelineRecordStoresInputHash -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run the full test suite to check for regressions** - -``` -uv run pytest tests/ -x -q -``` - -Expected: All existing tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/nodes/test_function_node_empty_data.py -git commit -m "feat(function_node): store INPUT_DATA_HASH_COL in pipeline DB tag table (ITL-534)" -``` - ---- - -## Task 7: Extend `_JoinedRecords` and `_fetch_joined_records` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines 678–697 `_JoinedRecords`, lines 1709–1855 `_fetch_joined_records`) -- Test: `tests/test_core/nodes/test_function_node_empty_data.py` (extend) - -- [ ] **Step 1: Add failing tests** - -Append to `tests/test_core/nodes/test_function_node_empty_data.py`: - -```python -from orcapod.core.datagrams.tag_data import EmptyData -from orcapod.core.nodes.function_node import _PIPELINE_BASE_ENTRY_ID_COL -from orcapod.types import NodeConfig - - -@pytest.fixture -def ephemeral_node(): - """FunctionJobNode with is_result_ephemeral=True and an ephemeral store.""" - table = pa.table({ - "key": pa.array(["a"], type=pa.large_string()), - "value": pa.array([1], type=pa.int64()), - }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) - node = FunctionJobNode( - pod, src, - pipeline_database=InMemoryArrowDatabase(), - result_database=None, - node_config=NodeConfig(is_result_ephemeral=True), - ) - node.set_ephemeral_store(InMemoryArrowDatabase()) - return node - - -class TestFetchJoinedRecordsEmptyData: - def test_returns_empty_data_tokens_field(self, ephemeral_node): - """_JoinedRecords now has an empty_data_tokens field (even when empty).""" - node = ephemeral_node - # Execute so pipeline DB has a record - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - result = node._fetch_joined_records() - assert result is not None - assert hasattr(result, "empty_data_tokens") - assert hasattr(result, "empty_taginfo_rows") - - def test_ephemeral_miss_produces_empty_data_token(self, ephemeral_node): - """When the ephemeral store is cleared, _fetch_joined_records emits EmptyData.""" - node = ephemeral_node - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - - # Simulate ephemeral result expiry by replacing the ephemeral store with an empty one - node.set_ephemeral_store(InMemoryArrowDatabase()) - - result = node._fetch_joined_records() - assert result is not None - assert len(result.empty_data_tokens) == 1 - - token = next(iter(result.empty_data_tokens.values())) - assert isinstance(token, EmptyData) - assert token.cached_content_hash is not None - - def test_empty_data_token_hash_matches_input(self, ephemeral_node): - """The EmptyData token's cached hash matches the original input's content hash.""" - node = ephemeral_node - input_pairs = list(node._input_stream.iter_data()) - tag0, data0 = input_pairs[0] - node.execute_data(tag0, data0) - - # Clear ephemeral store - node.set_ephemeral_store(InMemoryArrowDatabase()) - - result = node._fetch_joined_records() - assert result is not None - token = next(iter(result.empty_data_tokens.values())) - assert token.cached_content_hash.to_string() == data0.content_hash().to_string() - - def test_old_format_row_warns_and_produces_none_hash( - self, ephemeral_node, caplog - ): - """An ephemeral tag row without INPUT_DATA_HASH_COL logs a warning, hash=None.""" - import logging - node = ephemeral_node - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - - # Manually delete INPUT_DATA_HASH_COL from the pipeline DB to simulate old format - path = node.node_identity_path - all_records = node._pipeline_database.get_all_records(path) - assert all_records is not None - stripped = all_records.drop([constants.INPUT_DATA_HASH_COL]) - node._pipeline_database._tables[path] = stripped # InMemoryArrowDatabase internals - - # Clear ephemeral store to force an ephemeral miss - node.set_ephemeral_store(InMemoryArrowDatabase()) - - with caplog.at_level(logging.WARNING): - result = node._fetch_joined_records() - - assert result is not None - assert len(result.empty_data_tokens) == 1 - token = next(iter(result.empty_data_tokens.values())) - assert token.cached_content_hash is None - assert constants.INPUT_DATA_HASH_COL in caplog.text -``` - -Note: the last test accesses `InMemoryArrowDatabase._tables` directly. Run the full test first — if `InMemoryArrowDatabase` doesn't expose `_tables`, look at its implementation and use whatever internal attribute it uses to store tables. - -- [ ] **Step 2: Run to verify they fail** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestFetchJoinedRecordsEmptyData -v -``` - -Expected: `AttributeError` on `empty_data_tokens` or similar. - -- [ ] **Step 3: Extend `_JoinedRecords`** - -In `function_node.py`, replace the existing `_JoinedRecords` definition (around line 678): - -```python -class _JoinedRecords(NamedTuple): - """Internal result type returned by ``_fetch_joined_records``. - - Attributes: - table: The joined ``pa.Table`` for rows that matched in the result DB. - Always includes a ``_PIPELINE_BASE_ENTRY_ID_COL`` column. - taginfo_columns: Column names from the pipeline database fetch, - captured before the join. Used by ``_load_cached_entries`` to - derive tag keys in the CACHE_ONLY fallback path. - empty_data_tokens: Mapping from ``base_entry_id`` bytes to an - ``EmptyData`` token for each ephemeral tag row whose result - was not found in either store (cross-session miss). Empty dict - when no ephemeral misses occurred. - empty_taginfo_rows: Mapping from ``base_entry_id`` bytes to the raw - taginfo row dict for each entry in ``empty_data_tokens``. Used - by ``_load_cached_entries`` to reconstruct the tag for each token. - """ - - table: "pa.Table" - taginfo_columns: tuple[str, ...] - empty_data_tokens: "dict[bytes, EmptyData]" - empty_taginfo_rows: "dict[bytes, dict]" -``` - -Add `EmptyData` to the imports at the top of `function_node.py`. Find the existing `from orcapod.core.cached_function_pod import CachedFunctionPod` line and add below it: - -```python -from orcapod.core.datagrams.tag_data import EmptyData -from orcapod.errors import EphemeralResultMissingError -``` - -- [ ] **Step 4: Update `_fetch_joined_records` to emit `EmptyData` tokens** - -In `_fetch_joined_records` (around line 1709), find the ephemeral join section. Currently it ends with the comment `# Cross-session miss: eph_results is None → silently drop ephemeral entries`. Replace the entire ephemeral-join block with: - -```python - # ------------------------------------------------------------------ - # Ephemeral join - # ------------------------------------------------------------------ - ephemeral_df = pl.DataFrame() - empty_data_tokens: dict[bytes, EmptyData] = {} - empty_taginfo_rows: dict[bytes, dict] = {} - - if ephemeral_taginfo_df.height > 0: - eph_results = None - if self._ephemeral_cached_pod is not None: - eph_results = self._ephemeral_cached_pod.result_database.get_all_records( - self._ephemeral_cached_pod.record_path, - record_id_column=constants.DATA_RECORD_ID, - ) - if eph_results is not None: - if results_schema is None: - results_schema = eph_results.schema - ephemeral_df = ephemeral_taginfo_df.join( - pl.DataFrame(eph_results), - on=constants.DATA_RECORD_ID, - how="inner", - ) - - # Emit EmptyData tokens for unmatched ephemeral rows - # (cross-session miss: ephemeral data is gone). - unmatched_df = ephemeral_taginfo_df.join( - pl.DataFrame(eph_results) if eph_results is not None else pl.DataFrame(), - on=constants.DATA_RECORD_ID, - how="anti", - ) - for row in unmatched_df.iter_rows(named=True): - base_eid = row[_PIPELINE_BASE_ENTRY_ID_COL] - raw_hash = row.get(constants.INPUT_DATA_HASH_COL) - if raw_hash is None: - logger.warning( - "Pipeline DB row missing %r column — EmptyData will have " - "no cached hash; flow-through unavailable for this row. " - "base_entry_id: %r", - constants.INPUT_DATA_HASH_COL, - base_eid, - ) - cached_hash = None - else: - from orcapod.types import ContentHash as _ContentHash - cached_hash = _ContentHash.from_string(raw_hash) - empty_data_tokens[base_eid] = EmptyData( - cached_content_hash=cached_hash, - data_context=self.data_context, - ) - empty_taginfo_rows[base_eid] = row -``` - -Also update the empty-result return at the bottom of `_fetch_joined_records` to include the new fields: - -```python - # No results found in either store — return empty table preserving taginfo schema - empty_table = taginfo.slice(0, 0) - return _JoinedRecords( - table=empty_table, - taginfo_columns=taginfo_columns, - empty_data_tokens=empty_data_tokens, - empty_taginfo_rows=empty_taginfo_rows, - ) -``` - -And update the final return at the bottom of `_fetch_joined_records`: - -```python - return _JoinedRecords( - table=joined, - taginfo_columns=taginfo_columns, - empty_data_tokens=empty_data_tokens, - empty_taginfo_rows=empty_taginfo_rows, - ) -``` - -Also update the persistent-only-result early return (where `persistent_df` and `ephemeral_df` are both empty after the merge section): - -Search for `return _JoinedRecords(table=empty_table, taginfo_columns=taginfo_columns)` and any other `_JoinedRecords(...)` calls in `_fetch_joined_records` — add `empty_data_tokens=empty_data_tokens, empty_taginfo_rows=empty_taginfo_rows` to each. - -- [ ] **Step 5: Run the new tests** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestFetchJoinedRecordsEmptyData -v -``` - -Expected: PASS. If the old-format-row test fails due to `InMemoryArrowDatabase._tables` not being accessible, read `src/orcapod/databases/in_memory.py` and adjust the test to use the correct internal attribute. - -- [ ] **Step 6: Run existing fetch-joined tests to check for regressions** - -``` -uv run pytest tests/test_core/nodes/test_function_node_fetch_joined.py -v -``` - -Expected: All PASS (the new `empty_data_tokens` and `empty_taginfo_rows` fields default to empty dicts, which is backward-compatible). - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py tests/test_core/nodes/test_function_node_empty_data.py -git commit -m "feat(function_node): emit EmptyData tokens for ephemeral misses in _fetch_joined_records (ITL-534)" -``` - ---- - -## Task 8: Update `_load_cached_entries` to merge `EmptyData` tokens - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (lines 1856–1920, `_load_cached_entries`) -- Test: `tests/test_core/nodes/test_function_node_empty_data.py` (extend) - -- [ ] **Step 1: Add a failing test** - -Append to `tests/test_core/nodes/test_function_node_empty_data.py`: - -```python -class TestLoadCachedEntriesEmptyData: - def test_empty_data_token_appears_in_loaded_entries(self, ephemeral_node): - """_load_cached_entries yields (tag, EmptyData) for ephemeral miss rows.""" - node = ephemeral_node - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - - # Simulate expiry - node.set_ephemeral_store(InMemoryArrowDatabase()) - - loaded = node._load_cached_entries() - assert len(loaded) == 1 - base_eid = next(iter(loaded)) - tag_out, data_out = loaded[base_eid] - assert isinstance(data_out, EmptyData) - assert data_out.cached_content_hash is not None - - def test_normal_result_wins_over_empty_data_token(self, persistent_node): - """Non-ephemeral rows produce real Data, not EmptyData.""" - node = persistent_node - for tag, data in node._input_stream.iter_data(): - node.execute_data(tag, data) - - loaded = node._load_cached_entries() - for base_eid, (tag, data) in loaded.items(): - assert not isinstance(data, EmptyData) - assert isinstance(data, Data) -``` - -- [ ] **Step 2: Run to verify they fail** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestLoadCachedEntriesEmptyData -v -``` - -Expected: FAIL — `_load_cached_entries` returns `{}` when `table.num_rows == 0`. - -- [ ] **Step 3: Update `_load_cached_entries`** - -In `function_node.py`, replace the `_load_cached_entries` method (around line 1856) with: - -```python - def _load_cached_entries( - self, - base_entry_ids: list[bytes] | None = None, - ) -> "dict[bytes, tuple[TagProtocol, DataProtocol]]": - """DB loader: fetch ``(tag, data)`` pairs from the pipeline and result databases. - - Calls ``_fetch_joined_records`` to obtain the raw joined table, then - converts each row into a ``(tag, data)`` tuple keyed by base entry ID. - Also merges ``EmptyData`` tokens for ephemeral miss rows. - - If ``base_entry_ids`` is given, only those entries are fetched from DB. - If ``None``, all records for this node are loaded. - - Does NOT read from or write to the in-memory cache - (``_cached_output_datas``). Callers that want to populate the cache - must call ``self._cached_output_datas.update(loaded)`` themselves. - - Args: - base_entry_ids: If provided, load only these specific base entry IDs. - If ``None``, load all records for this node. - - Returns: - dict mapping base_entry_id → ``(tag, data)``. Empty dict when - either database is absent, both fetches return nothing, and no - ``EmptyData`` tokens exist. - """ - from orcapod.core.datagrams.tag_data import Tag - - fetched = self._fetch_joined_records(base_entry_ids=base_entry_ids) - if fetched is None: - return {} - if fetched.table.num_rows == 0 and not fetched.empty_data_tokens: - return {} - - joined = fetched.table - - # Derive tag keys: prefer input_stream when available; fall back to - # taginfo column exclusion for CACHE_ONLY / deserialized nodes. - if self._input_stream is not None: - tag_keys = self._input_stream.keys()[0] - else: - tag_keys = tuple( - c - for c in fetched.taginfo_columns - if not c.startswith(constants.META_PREFIX) - and not c.startswith(constants.SOURCE_PREFIX) - and not c.startswith(constants.SYSTEM_TAG_PREFIX) - and c != _PIPELINE_ENTRY_ID_COL - and c != constants.NODE_CONTENT_HASH_COL - ) - - loaded: "dict[bytes, tuple[TagProtocol, DataProtocol]]" = {} - - # --- Process normal (fully matched) rows --- - if joined.num_rows > 0: - base_entry_ids_col = joined.column(_PIPELINE_BASE_ENTRY_ID_COL).to_pylist() - drop_cols = [ - c - for c in joined.column_names - if c.startswith(constants.META_PREFIX) - or c == constants.NODE_CONTENT_HASH_COL - ] - data_table = joined.drop([c for c in drop_cols if c in joined.column_names]) - stream = ArrowTableStream(data_table, tag_columns=tag_keys) - for base_eid, (tag, data) in zip(base_entry_ids_col, stream.iter_data()): - loaded[base_eid] = (tag, data) - - # --- Process EmptyData tokens (ephemeral misses) --- - for base_eid, empty_data in fetched.empty_data_tokens.items(): - if base_eid in loaded: - # A persistent result exists for the same entry — it wins. - continue - raw_row = fetched.empty_taginfo_rows[base_eid] - tag_data = {k: v for k, v in raw_row.items() if k in tag_keys} - system_tags = { - k: v - for k, v in raw_row.items() - if k.startswith(constants.SYSTEM_TAG_PREFIX) - } - tag = Tag( - tag_data, system_tags=system_tags, data_context=self.data_context - ) - loaded[base_eid] = (tag, empty_data) - - return loaded -``` - -- [ ] **Step 4: Run the new tests** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestLoadCachedEntriesEmptyData -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run the full node test suite** - -``` -uv run pytest tests/test_core/nodes/ -v -q -``` - -Expected: All PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "feat(function_node): merge EmptyData tokens in _load_cached_entries (ITL-534)" -``` - ---- - -## Task 9: Downstream guard in `_process_data_internal` and async counterpart - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` (`_process_data_internal` line 1198, `_async_process_data_internal` line 1323) -- Test: `tests/test_core/nodes/test_function_node_empty_data.py` (extend) - -- [ ] **Step 1: Add failing tests** - -Append to `tests/test_core/nodes/test_function_node_empty_data.py`: - -```python -from orcapod.errors import EphemeralResultMissingError - - -@pytest.fixture -def downstream_node(ephemeral_node): - """Persistent downstream FunctionJobNode that follows ephemeral_node.""" - table = pa.table({ - "key": pa.array(["a"], type=pa.large_string()), - "value": pa.array([1], type=pa.int64()), - }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - - def triple_value(result: int) -> int: - return result * 3 - - pod = FunctionPod(PythonDataFunction(triple_value, output_keys="final")) - return FunctionJobNode( - pod, ephemeral_node, - pipeline_database=InMemoryArrowDatabase(), - result_database=InMemoryArrowDatabase(), - ) - - -class TestProcessDataInternalEmptyData: - def test_cache_hit_returns_cached_result(self, ephemeral_node, downstream_node): - """Downstream cache hit on EmptyData input returns the cached result.""" - # Step 1: Run upstream and downstream together (both have live data) - for tag, data in ephemeral_node._input_stream.iter_data(): - ephemeral_node.execute_data(tag, data) - for tag, data in ephemeral_node.iter_data(): - downstream_node.execute_data(tag, data) - - # Step 2: Simulate upstream ephemeral expiry - ephemeral_node.set_ephemeral_store(InMemoryArrowDatabase()) - ephemeral_node._cached_output_datas.clear() - - # Step 3: Downstream receives EmptyData from upstream's iter_data - upstream_items = list(ephemeral_node.iter_data()) - assert len(upstream_items) == 1 - tag_in, data_in = upstream_items[0] - assert isinstance(data_in, EmptyData) - - # Step 4: Downstream processes the EmptyData — should hit its result cache - tag_out, result = downstream_node.execute_data(tag_in, data_in) - assert result is not None - assert result.as_dict()["final"] == 6 # double(1)=2, triple(2)=6 - - def test_cache_miss_raises_ephemeral_result_missing_error(self, ephemeral_node): - """Downstream cache miss on EmptyData input raises EphemeralResultMissingError.""" - # Run upstream but NOT downstream - for tag, data in ephemeral_node._input_stream.iter_data(): - ephemeral_node.execute_data(tag, data) - - # Simulate expiry - ephemeral_node.set_ephemeral_store(InMemoryArrowDatabase()) - ephemeral_node._cached_output_datas.clear() - - # Build a downstream node - table = pa.table({ - "key": pa.array(["a"], type=pa.large_string()), - "value": pa.array([1], type=pa.int64()), - }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - - def triple_value(result: int) -> int: - return result * 3 - - pod = FunctionPod(PythonDataFunction(triple_value, output_keys="final")) - downstream = FunctionJobNode( - pod, ephemeral_node, - pipeline_database=InMemoryArrowDatabase(), - result_database=InMemoryArrowDatabase(), - ) - - upstream_items = list(ephemeral_node.iter_data()) - tag_in, data_in = upstream_items[0] - assert isinstance(data_in, EmptyData) - - with pytest.raises(EphemeralResultMissingError) as exc_info: - downstream.execute_data(tag_in, data_in) - - assert exc_info.value.cached_content_hash is not None - assert exc_info.value.node_identity_path == downstream.node_identity_path -``` - -- [ ] **Step 2: Run to verify they fail** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestProcessDataInternalEmptyData -v -``` - -Expected: FAIL — `EmptyDataAccessError` raised when downstream tries to compute with `EmptyData`. - -- [ ] **Step 3: Add `EmptyData` guard at the top of `_process_data_internal`** - -In `function_node.py`, find `_process_data_internal` (around line 1198). Insert an `isinstance` guard as the first action inside the method body, before the `ephemeral_result` check: - -```python - def _process_data_internal( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - # Guard: EmptyData inputs skip computation and go straight to cache lookup. - # EmptyData.content_hash() returns the cached hash, so result_cache.lookup() - # works transparently. On cache miss, fail loudly — the ephemeral data is - # gone and the downstream has not yet computed a result for this input. - if isinstance(data, EmptyData): - if self._cached_function_pod is not None: - cached = self._cached_function_pod.result_cache.lookup(data) - if cached is not None: - cached = cached.with_meta_columns( - **{self._cached_function_pod.RESULT_COMPUTED_FLAG: False} - ) - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=cached.datagram_uuid, - computed=False, - ) - base_entry_id = self.compute_base_entry_id(tag, data) - self._cached_output_datas[base_entry_id] = (tag, cached) - self._cached_output_table = None - self._cached_content_hash_column = None - return tag, cached - raise EphemeralResultMissingError( - tag=tag, - cached_content_hash=data.cached_content_hash, - node_identity_path=self.node_identity_path, - message=( - "Downstream cache miss for EmptyData input — " - "ephemeral result is gone and downstream has not yet computed " - "a result for this input hash." - ), - ) - - # ... rest of existing method unchanged ... - ephemeral_result = self._node_config.is_result_ephemeral or False - ... -``` - -- [ ] **Step 4: Add the identical guard to `_async_process_data_internal`** - -Find `_async_process_data_internal` (around line 1323) and insert the same guard, but with `await` for the async path. The cache lookup itself is synchronous (ResultCache is sync), so no `await` is needed for the lookup: - -```python - async def _async_process_data_internal( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - # Guard: same as sync _process_data_internal. - if isinstance(data, EmptyData): - if self._cached_function_pod is not None: - cached = self._cached_function_pod.result_cache.lookup(data) - if cached is not None: - cached = cached.with_meta_columns( - **{self._cached_function_pod.RESULT_COMPUTED_FLAG: False} - ) - if self._pipeline_database is not None: - self.add_pipeline_record( - tag, - data, - data_record_id=cached.datagram_uuid, - computed=False, - ) - base_entry_id = self.compute_base_entry_id(tag, data) - self._cached_output_datas[base_entry_id] = (tag, cached) - self._cached_output_table = None - self._cached_content_hash_column = None - return tag, cached - raise EphemeralResultMissingError( - tag=tag, - cached_content_hash=data.cached_content_hash, - node_identity_path=self.node_identity_path, - message=( - "Downstream cache miss for EmptyData input — " - "ephemeral result is gone and downstream has not yet computed " - "a result for this input hash." - ), - ) - - # ... rest of existing async method unchanged ... - ephemeral_result = self._node_config.is_result_ephemeral or False - ... -``` - -- [ ] **Step 5: Run the new tests** - -``` -uv run pytest tests/test_core/nodes/test_function_node_empty_data.py::TestProcessDataInternalEmptyData -v -``` - -Expected: PASS. - -- [ ] **Step 6: Run the full test suite** - -``` -uv run pytest tests/ -x -q -``` - -Expected: All PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "feat(function_node): add EmptyData guard in _process_data_internal (ITL-534)" -``` - ---- - -## Task 10: Create follow-up Linear issues - -Before creating the PR, create the three follow-up issues using the Linear MCP tool. Use the project ID `0f424f4d-74d3-4efd-aed6-b0364a7368fb` (Orcapod Python v0.2 Feature Sprint) and team `ITL`. - -- [ ] **Step 1: Create issue — Configurable EmptyData relaxation** - -``` -mcp__claude_ai_Linear__save_issue( - title: "EmptyData: configurable relaxation of strict upstream-ephemerality enforcement", - team: "ITL", - project: "0f424f4d-74d3-4efd-aed6-b0364a7368fb", - description: """ -## Overview -ITL-534 treats any EmptyData arriving at a downstream pod as a legitimate -ephemeral miss (option A). This follow-up adds per-pipeline opt-in to allow -graceful treatment of EmptyData from non-ephemeral upstreams (options B/C -from the original design brainstorm). - -## Goals & Success Criteria -* A pipeline-level flag (or NodeConfig field) allows downstream pods to - accept EmptyData from non-ephemeral upstreams without raising immediately. -* Default behavior (strict) is unchanged. -* Configurable relaxation is documented and tested. - -## Scope & Boundaries -In scope: -* Per-pipeline or per-node config flag for EmptyData handling mode. -* Documentation of when to use each mode. -Out of scope: -* Changes to EmptyData data model (already done in ITL-534). - -## Dependencies & Risks -* Depends on ITL-534 landing first. -""" -) -``` - -- [ ] **Step 2: Create issue — Rigorous upstream-ephemerality validation** - -``` -mcp__claude_ai_Linear__save_issue( - title: "EmptyData: rigorous upstream-ephemerality validation in _process_data_internal", - team: "ITL", - project: "0f424f4d-74d3-4efd-aed6-b0364a7368fb", - description: """ -## Overview -ITL-534 treats any EmptyData as a legitimate ephemeral miss. This follow-up -adds a check that EmptyData only arrives from upstream pods declared with -is_result_ephemeral=True, failing loudly if EmptyData arrives from a -non-ephemeral upstream (indicating data corruption). - -## Goals & Success Criteria -* _process_data_internal inspects the source of EmptyData. -* EmptyData from a non-ephemeral upstream raises a distinct exception. -* Legitimate ephemeral misses continue to work as before. - -## Scope & Boundaries -In scope: -* Source-ephemerality check in _process_data_internal. -* Distinct exception for corruption vs. expected miss. -Out of scope: -* Changes to EmptyData data model. - -## Dependencies & Risks -* Depends on ITL-534. -* Requires the empty_source_info field to carry ephemerality provenance. -""" -) -``` - -- [ ] **Step 3: Create issue — Tag-row reconstruction from downstream cache evidence** - -``` -mcp__claude_ai_Linear__save_issue( - title: "EmptyData: tag-row reconstruction from downstream cache evidence", - team: "ITL", - project: "0f424f4d-74d3-4efd-aed6-b0364a7368fb", - description: """ -## Overview -ITL-534 defines the empty_source_info field on EmptyData but defers the -reconstruction logic. This follow-up implements tag-row reconstruction: -when a downstream cache hit is found for an EmptyData input, write a -reconstructed upstream tag row with record_id=None as a provenance marker. - -## Goals & Success Criteria -* Downstream cache hit against EmptyData writes a reconstructed tag row. -* Reconstructed rows have record_id=None in the source-column set. -* Inspection tooling can distinguish reconstructed rows from normal rows. - -## Scope & Boundaries -In scope: -* populate empty_source_info on EmptyData tokens. -* Write reconstructed tag rows with record_id=None. -* Resolve extension axes 3-6 from ITL-534 description before landing. -Out of scope: -* Cross-pipeline reconstruction (axis 6) — separate design required. - -## Dependencies & Risks -* Depends on ITL-534. -* Extension axes 5 (orphaned reconstructed rows) and 6 (cross-pipeline - trust) must be resolved before this lands. -""" -) -``` - -- [ ] **Step 4: Verify the three issues were created and note their IDs** - -The tool response will include issue IDs. Record them. - ---- - -## Task 11: Final verification and PR - -- [ ] **Step 1: Run the full test suite** - -``` -uv run pytest tests/ -q -``` - -Expected: All PASS. - -- [ ] **Step 2: Create the PR** - -```bash -gh pr create \ - --title "feat(datagrams,function_node): add EmptyData + ephemeral result propagation (ITL-534)" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Adds `EmptyData(Data)` — a payload-free `Data` subclass carrying an optional cached content hash, used to represent expired/pruned ephemeral results in the pipeline. -- Extends the tag table (pipeline DB) to store `INPUT_DATA_HASH_COL` in every row written by `add_pipeline_record()`. -- `_fetch_joined_records()` now emits `EmptyData` tokens (via anti-join) for ephemeral miss rows instead of silently dropping them. -- `_load_cached_entries()` merges `EmptyData` tokens with normal rows, so `iter_data()` propagates them downstream. -- `_process_data_internal()` and `_async_process_data_internal()` add an `isinstance(data, EmptyData)` guard: cache-only lookup via `result_cache.lookup(empty_data)` on hit returns the cached result; on miss raises `EphemeralResultMissingError` loudly. -- Three new exception types: `EmptyDataAccessError`, `EmptyDataHashMissingError`, `EphemeralResultMissingError`. -- Old-format pipeline DB rows (lacking `INPUT_DATA_HASH_COL`) produce a WARNING and an `EmptyData` with `cached_content_hash=None`; no forced migration. - -Closes ITL-534 - -## Test plan - -- [ ] `uv run pytest tests/test_core/test_empty_data.py -v` — unit tests for `EmptyData` and exceptions -- [ ] `uv run pytest tests/test_core/test_result_cache.py -v` — lookup-via-`EmptyData` tests -- [ ] `uv run pytest tests/test_core/nodes/test_function_node_empty_data.py -v` — integration tests (write path, read path, downstream guard) -- [ ] `uv run pytest tests/ -q` — full suite, no regressions - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - -- [ ] **Step 3: Confirm PR URL is returned and report it** - ---- - -## Self-Review Checklist - -**Spec coverage:** - -| Spec requirement | Task | -|---|---| -| `EmptyData(Data)` with `cached_content_hash` + `empty_source_info` | Task 2 | -| `EmptyDataAccessError`, `EmptyDataHashMissingError`, `EphemeralResultMissingError` | Task 1 | -| `INPUT_DATA_HASH_COL` in `add_pipeline_record()` | Task 6 | -| `_fetch_joined_records()` emits `EmptyData` tokens | Task 7 | -| `_process_data_internal()` + async guard | Task 9 | -| `_load_cached_entries()` merges tokens | Task 8 | -| `_JoinedRecords` extended | Task 7 | -| Warning log for old-format rows | Task 7 | -| Unit tests for `EmptyData` | Task 2 | -| `ResultCache.lookup` via `EmptyData` tests | Task 5 | -| Integration tests (happy path, miss, old-format) | Task 9 | - -**Type consistency:** `EmptyData` defined in Task 2, used in Tasks 7–9. `EphemeralResultMissingError` defined in Task 1, raised in Task 9. `result_cache` property defined in Task 4, used in Task 9. All consistent. - -**Placeholders:** None. diff --git a/superpowers/plans/2026-07-16-itl-532-side-effect-function-pod.md b/superpowers/plans/2026-07-16-itl-532-side-effect-function-pod.md deleted file mode 100644 index baf3d5c50..000000000 --- a/superpowers/plans/2026-07-16-itl-532-side-effect-function-pod.md +++ /dev/null @@ -1,2331 +0,0 @@ -# SideEffectFunctionPod Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement `SideEffectFunctionPod/Node` — a hybrid pod wrapping `(ctx: InvocationContext, arg1, ...) -> OutputData`, producing a downstream data stream while receiving per-row `InvocationContext`. Includes a breaking change to `SideEffectPod`'s callable contract. - -**Architecture:** `SideEffectFunctionPod` is a `TraceableBase` that strips the `ctx` parameter from function signature for schema inference (via `_strip_ctx_from_fn`), passes data fields as keyword arguments (like `FunctionPod`), and injects `InvocationContext` per-row. Pipeline integration mirrors `FunctionPod` (records invocations, compiles to `SideEffectFunctionJobNode`) while execution mirrors `SideEffectPod` (same preimage construction, invocation logging via `_write_invocation_row`). - -**Tech Stack:** Python 3.11+, PyArrow, orcapod core (TraceableBase, StreamBase, ResultCache, InvocationContext, _write_invocation_row), uuid_utils, uv, pytest. - ---- - -## File Map - -| File | Change | -|------|--------| -| `src/orcapod/side_effects.py` | Modify — add `ctx_arg_name`, change call style to `**kwargs` | -| `src/orcapod/core/side_effect_function/__init__.py` | New | -| `src/orcapod/core/side_effect_function/side_effect_function_pod.py` | New — all classes + decorator | -| `src/orcapod/protocols/core_protocols/side_effect_function_pod.py` | New — protocol | -| `src/orcapod/protocols/core_protocols/__init__.py` | Modify — export | -| `src/orcapod/protocols/core_protocols/trackers.py` | Modify — new method | -| `src/orcapod/core/tracker.py` | Modify — new method | -| `src/orcapod/pipeline/pod_invocation.py` | Modify — new invocation type | -| `src/orcapod/pipeline/base.py` | Modify — recorder + compile + to_invocations | -| `src/orcapod/protocols/node_protocols.py` | Modify — new protocol + TypeGuard | -| `src/orcapod/pipeline/sync_orchestrator.py` | Modify — new elif branch | -| `src/orcapod/pipeline/async_orchestrator.py` | Modify — new elif branch | -| `src/orcapod/pipeline/graph.py` | Modify — class attribute | -| `src/orcapod/pipeline/job.py` | Modify — class attr + _distribute_databases + as_pipeline | -| `src/orcapod/__init__.py` | Modify — re-exports | -| `tests/test_core/side_effect_pod/test_side_effect_pod.py` | Modify — update for new call style | -| `tests/test_core/side_effect_function/__init__.py` | New | -| `tests/test_core/side_effect_function/test_side_effect_function_pod.py` | New — SF-01–SF-13 | - ---- - -### Task 1: SideEffectPod Breaking Change — `ctx_arg_name` + New Call Style - -Update `side_effects.py` to add `ctx_arg_name: str = "ctx"` and change the user function call from `fn(data, ctx)` to `fn(**{ctx_arg_name: ctx, **data.as_dict()})`. Update all existing tests to match. - -**Files:** -- Modify: `src/orcapod/side_effects.py` -- Modify: `tests/test_core/side_effect_pod/test_side_effect_pod.py` - -- [ ] **Step 1: Update all existing test functions from `fn(data, ctx)` to keyword-arg style** - -The test stream `_make_stream(n)` has one data column `value: int64`. After the change, `fn` is called as `fn(ctx=ctx, value=42)` (via `fn(**{ctx_arg_name: ctx, **data.as_dict()})`). Update every `def fn(data, ctx)` pattern in the test file: - -```python -# BEFORE — every test function using the old call style: -def fn(data, ctx): - calls.append(dict(data)) - -# AFTER — receives individual keyword arguments: -def fn(value, ctx): - calls.append({"value": value}) - -# BEFORE — pass-through ignore ctx: -def fn(data, _ctx): - calls.append(True) - -# AFTER: -def fn(value, _ctx): - calls.append(True) - -# BEFORE — raise inside fn: -def fn(data, ctx): - raise RuntimeError("boom") - -# AFTER: -def fn(value, ctx): - raise RuntimeError("boom") - -# BEFORE — T3 receive ctx: -def fn(data, ctx): - received.append(ctx) - -# AFTER: -def fn(value, ctx): - received.append(ctx) -``` - -Apply this transformation to ALL test methods that define inline `fn` callables: `test_t1_*`, `test_t2_*`, `test_t3_*`, `test_t4_*`, `test_t8_*`, `test_t9_*`, `test_t10_*`, `test_t11_*`, `test_t12_*`, `test_t13_*`, `test_t14_*`, `test_t15_*`, and all decorator tests (T16-T18). Also update `test_failure_does_not_write_log_*` and `test_fail_then_succeed_then_skip` (their `fn(data, ctx)` patterns). For `_make_node_with_db` helper, update its inner `fn` too. - -Also update `TestSideEffectInvocation.test_construction`: change `def fn(data, ctx): pass` → `def fn(value, ctx): pass`. - -Add two new test classes at the bottom of the file for the new call style and `ctx_arg_name`: - -```python -class TestSideEffectPodNewCallStyle: - """SEP-UPDATE: Data fields passed as kwargs, not as DataProtocol.""" - - def test_sep_update_kwargs_passed_to_fn(self): - """Data fields unpacked as kwargs; ctx passed by ctx_arg_name.""" - from orcapod.side_effects import SideEffectPod - - received: dict = {} - - def fn(value, ctx): - received["value"] = value - received["ctx"] = ctx - - pod = SideEffectPod(fn) - stream = _make_stream(1) # stream has 'value' data column (int64) - list(pod.process(stream).iter_data()) - - assert "value" in received - assert isinstance(received["value"], int) - assert received["ctx"] is not None - - -class TestSideEffectPodCtxArgName: - """SEP-CTX-NAME: ctx_arg_name routes InvocationContext to correct parameter.""" - - def test_sep_ctx_name_custom(self): - """ctx_arg_name='context' injects InvocationContext under that name.""" - from orcapod.side_effects import SideEffectPod, InvocationContext - - received: dict = {} - - def fn(value, context): - received["context"] = context - - pod = SideEffectPod(fn, ctx_arg_name="context") - stream = _make_stream(1) - list(pod.process(stream).iter_data()) - - assert isinstance(received["context"], InvocationContext) -``` - -- [ ] **Step 2: Run tests to confirm they fail (tests reference old call style)** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py -x -q 2>&1 | head -40 -``` - -Expected: `TypeError` — existing functions like `fn(data, ctx)` cannot receive `value=...` as a keyword argument. - -- [ ] **Step 3: Add `ctx_arg_name` parameter to `SideEffectPod.__init__`** - -In `src/orcapod/side_effects.py`, update `SideEffectPod.__init__`: - -```python -def __init__( - self, - fn: Callable, - config: SideEffectPodConfig | None = None, - tracker_manager: TrackerManagerProtocol | None = None, - name: str | None = None, - label: str | None = None, - data_context: Any = None, - ctx_arg_name: str = "ctx", # NEW -) -> None: - super().__init__(label=label, data_context=data_context) - self._fn = fn - self._name: str = name if name is not None else getattr(fn, "__name__", "unknown") - self._pod_config = config or SideEffectPodConfig() - self.tracker_manager = tracker_manager or DEFAULT_TRACKER_MANAGER - self._ctx_arg_name: str = ctx_arg_name # NEW -``` - -- [ ] **Step 4: Update `SideEffectPod.identity_structure()` to include `ctx_arg_name`** - -```python -def identity_structure(self) -> Any: - return ( - self.uri, - self._ctx_arg_name, # NEW — renaming ctx param changes hash - self._pod_config.track_completion, - self._pod_config.drop_on_failure, - ) -``` - -- [ ] **Step 5: Update `_execute_side_effect_row` — add `ctx_arg_name` param, change fn call** - -Add `ctx_arg_name: str = "ctx"` to the function signature (after `arrow_hasher`): - -```python -def _execute_side_effect_row( - *, - fn: Callable, - tag: TagProtocol, - data: DataProtocol, - pod_config: SideEffectPodConfig, - pipeline_hash_ch: ContentHash, - node_content_hash_str: str, - pod_name: str, - run_id: str | None, - arrow_hasher: Any, - ctx_arg_name: str = "ctx", # NEW - pipeline_database: ArrowDatabaseProtocol | None = None, - table_path: tuple[str, ...] | None = None, -) -> tuple[TagProtocol, DataProtocol] | None: -``` - -Change the fn call in step 5 of the function body from: -```python -fn(data, ctx) -``` -to: -```python -fn(**{ctx_arg_name: ctx, **data.as_dict()}) -``` - -- [ ] **Step 6: Pass `ctx_arg_name` in all four callers of `_execute_side_effect_row`** - -```python -# 1. SideEffectPodStream.iter_data(): -result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - node_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=None, - arrow_hasher=self._pod.data_context.arrow_hasher, - ctx_arg_name=self._pod._ctx_arg_name, # NEW -) - -# 2. SideEffectNode.iter_data(): -result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - node_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=None, - arrow_hasher=self._pod.data_context.arrow_hasher, - ctx_arg_name=self._pod._ctx_arg_name, # NEW - pipeline_database=None, - table_path=None, -) - -# 3. SideEffectJobNode.execute(): -result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - node_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=run_id, - arrow_hasher=self._pod.data_context.arrow_hasher, - ctx_arg_name=self._pod._ctx_arg_name, # NEW - pipeline_database=self._pipeline_database, - table_path=self._table_path, -) - -# 4. SideEffectJobNode.async_execute() — process_one inner function: -result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - node_content_hash_str=self._pod.content_hash().to_string(), - pod_name=self._pod.label, - run_id=run_id, - arrow_hasher=self._pod.data_context.arrow_hasher, - ctx_arg_name=self._pod._ctx_arg_name, # NEW - pipeline_database=self._pipeline_database, - table_path=self._table_path, -) -``` - -- [ ] **Step 7: Add `ctx_arg_name` to the three decorators** - -```python -def side_effect_pod( - fn: Callable | None = None, - *, - config: SideEffectPodConfig | None = None, - name: str | None = None, - ctx_arg_name: str = "ctx", # NEW -) -> SideEffectPod | Callable[[Callable], SideEffectPod]: - def _wrap(f: Callable) -> SideEffectPod: - return SideEffectPod(f, config=config, name=name, ctx_arg_name=ctx_arg_name) - if fn is not None: - return _wrap(fn) - return _wrap - -# Apply the same pattern to sink_pod and tap_pod. -``` - -- [ ] **Step 8: Run tests to confirm all pass** - -```bash -uv run pytest tests/test_core/side_effect_pod/test_side_effect_pod.py -v -``` - -Expected: all tests pass including the two new `TestSideEffectPodNewCallStyle` and `TestSideEffectPodCtxArgName` tests. - -- [ ] **Step 9: Run full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -x -q --tb=short 2>&1 | tail -10 -``` - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/side_effects.py tests/test_core/side_effect_pod/test_side_effect_pod.py -git commit -m "feat(side_effects): add ctx_arg_name param, change call style to **kwargs (breaking)" -``` - ---- - -### Task 2: New Module Scaffold + `SideEffectFunctionPod` Core Class (SF-01–SF-03, SF-10) - -Create `src/orcapod/core/side_effect_function/` with all classes as stubs, implement the `SideEffectFunctionPod` constructor including `_strip_ctx_from_fn`, schema extraction, and URI. - -**Files:** -- Create: `src/orcapod/core/side_effect_function/__init__.py` -- Create: `src/orcapod/core/side_effect_function/side_effect_function_pod.py` -- Create: `tests/test_core/side_effect_function/__init__.py` -- Create: `tests/test_core/side_effect_function/test_side_effect_function_pod.py` (SF-01–SF-03, SF-10) - -- [ ] **Step 1: Create test file with SF-01, SF-02, SF-03, SF-10** - -Create `tests/test_core/side_effect_function/__init__.py` (empty). - -Create `tests/test_core/side_effect_function/test_side_effect_function_pod.py`: - -```python -# tests/test_core/side_effect_function/test_side_effect_function_pod.py -from __future__ import annotations - -import pytest -import pyarrow as pa - -from orcapod.core.streams import ArrowTableStream -from orcapod.side_effects import InvocationContext - - -def _make_stream(n: int = 3) -> ArrowTableStream: - """Simple stream: tag=id (int), data=value (int).""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("value", pa.int64(), nullable=False), - ]) - table = pa.table( - {"id": list(range(n)), "value": list(range(n))}, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -def _make_in_memory_db(): - """Return a fresh in-memory ArrowDatabase.""" - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase - return InMemoryArrowDatabase() - - -class TestSideEffectFunctionPodSchema: - """SF-01, SF-02, SF-03, SF-10: schema inference and ctx stripping.""" - - def test_sf01_ctx_stripped_from_input_schema(self): - """SF-01: 'ctx' param stripped; data params form the input schema.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod - - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"result_{value}" - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - - # Input schema excludes 'ctx' - assert "ctx" not in pod.input_data_schema - assert "value" in pod.input_data_schema - assert pod.input_data_schema["value"] == int - - # Output schema has the declared key - assert "result" in pod.output_data_schema - assert pod.output_data_schema["result"] == str - - def test_sf02_custom_ctx_arg_name(self): - """SF-02: ctx_arg_name='context' — stripped and injected by correct name.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod - - def my_fn(value: int, context: InvocationContext) -> str: - return f"r_{value}" - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"], ctx_arg_name="context") - assert "context" not in pod.input_data_schema - assert "value" in pod.input_data_schema - - def test_sf03_missing_ctx_arg_raises_at_construction(self): - """SF-03: Missing ctx_arg_name raises ValueError at construction time.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod - - def my_fn(value: int) -> str: - return str(value) - - with pytest.raises(ValueError, match="ctx_arg_name"): - SideEffectFunctionPod(my_fn, output_keys=["result"]) - # Default ctx_arg_name="ctx" is missing from my_fn's signature - - def test_sf10_node_uri_shape(self): - """SF-10: node_uri starts with 'side_effect_function' and has 5 elements.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod - - def my_fn(value: int, ctx: InvocationContext) -> str: - return str(value) - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - assert pod.uri[0] == "side_effect_function" - assert pod.uri[-1] == "python_side_effect_function" - assert len(pod.uri) == 5 - assert pod.uri[3] == "v1" -``` - -- [ ] **Step 2: Run tests to confirm they fail with ImportError** - -```bash -uv run pytest tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectFunctionPodSchema -x -q 2>&1 | head -20 -``` - -Expected: `ImportError` — module does not exist yet. - -- [ ] **Step 3: Create `src/orcapod/core/side_effect_function/__init__.py`** - -```python -"""SideEffectFunctionPod — function pod with per-row InvocationContext.""" -from orcapod.core.side_effect_function.side_effect_function_pod import ( - SideEffectFunctionPod, - SideEffectFunctionPodStream, - SideEffectFunctionNode, - SideEffectFunctionJobNode, - side_effect_function_pod, -) - -__all__ = [ - "SideEffectFunctionPod", - "SideEffectFunctionPodStream", - "SideEffectFunctionNode", - "SideEffectFunctionJobNode", - "side_effect_function_pod", -] -``` - -- [ ] **Step 4: Create `side_effect_function_pod.py` — implement `_strip_ctx_from_fn` and `SideEffectFunctionPod`** - -Create `src/orcapod/core/side_effect_function/side_effect_function_pod.py`: - -```python -"""SideEffectFunctionPod — hybrid of FunctionPod and SideEffectPod.""" -from __future__ import annotations - -import asyncio -import functools -import inspect -import logging -import sys -import uuid -from collections.abc import Callable, Collection, Iterator, Sequence -from typing import TYPE_CHECKING, Any - -from uuid_utils import uuid7 - -from orcapod.core.base import TraceableBase -from orcapod.core.streams.base import StreamBase -from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.errors import PipelineJobRequiredError -from orcapod.side_effects import ( - InvocationContext, - SideEffectPodConfig, - _SIDE_EFFECT_RECOMPUTATION_INDEX_COL, - _write_invocation_row, -) -from orcapod.utils.lazy_module import LazyModule -from orcapod.utils.schema_utils import extract_function_schemas - -if TYPE_CHECKING: - import pyarrow as pa - from orcapod.channels import ReadableChannel, WritableChannel - from orcapod.protocols.core_protocols import ( - DataProtocol, - StreamProtocol, - TagProtocol, - TrackerManagerProtocol, - ) - from orcapod.protocols.database_protocols import ArrowDatabaseProtocol - from orcapod.protocols.observability_protocols import ExecutionObserverProtocol - from orcapod.types import ColumnConfig, ContentHash, Schema -else: - pa = LazyModule("pyarrow") - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# _strip_ctx_from_fn — remove ctx param from signature for schema inference -# --------------------------------------------------------------------------- - - -def _strip_ctx_from_fn(fn: Callable, ctx_arg_name: str) -> Callable: - """Return a wrapper of ``fn`` with ``ctx_arg_name`` removed from signature. - - The wrapper is passed to ``extract_function_schemas`` so the context - parameter is transparent to schema inference. The original ``fn`` is used - for actual calls and content hashing. - - Args: - fn: The original user function. - ctx_arg_name: Name of the parameter receiving ``InvocationContext``. - - Returns: - A wrapper whose ``__signature__`` and ``__annotations__`` exclude - ``ctx_arg_name``. - - Raises: - ValueError: If ``ctx_arg_name`` is not in ``fn``'s signature. - """ - sig = inspect.signature(fn) - if ctx_arg_name not in sig.parameters: - raise ValueError( - f"ctx_arg_name {ctx_arg_name!r} not found in function signature " - f"{fn.__name__!r}. Available parameters: {list(sig.parameters)}" - ) - new_params = [p for n, p in sig.parameters.items() if n != ctx_arg_name] - new_sig = sig.replace(parameters=new_params) - - @functools.wraps(fn) - def wrapper(*args, **kwargs): # pragma: no cover - return fn(*args, **kwargs) - - wrapper.__signature__ = new_sig # type: ignore[attr-defined] - wrapper.__annotations__ = { - k: v for k, v in fn.__annotations__.items() if k != ctx_arg_name - } - return wrapper - - -# --------------------------------------------------------------------------- -# _build_ctx_and_record_id — shared preimage helper -# --------------------------------------------------------------------------- - - -def _build_ctx_and_record_id( - *, - pod: "SideEffectFunctionPod", - tag: "TagProtocol", - data: "DataProtocol", - pipeline_hash_ch: "ContentHash", - run_id: str | None, -) -> "tuple[InvocationContext, ContentHash, bytes]": - """Build InvocationContext + record ID for one (tag, data) row. - - Uses the same preimage as ``_execute_side_effect_row`` in - ``side_effects.py``: system-tag columns + INPUT_DATA_HASH_COL + - NODE_CONTENT_HASH_COL + recomputation index 0. - - Args: - pod: The invoking pod. - tag: Tag for this row. - data: Data for this row. - pipeline_hash_ch: Pipeline hash of the node. - run_id: Pipeline run identifier, or ``None`` in standalone mode. - - Returns: - ``(ctx, record_id_hash, record_id)`` where ``record_id`` is the - prefixed digest of ``record_id_hash``. - """ - from orcapod.system_constants import constants - - preimage = ( - tag.as_table(columns={"system_tags": True}) - .append_column( - constants.INPUT_DATA_HASH_COL, - pa.array([data.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - constants.NODE_CONTENT_HASH_COL, - pa.array([pod.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - _SIDE_EFFECT_RECOMPUTATION_INDEX_COL, - pa.array([0], type=pa.int32()), - ) - ) - record_id_hash: ContentHash = pod.data_context.arrow_hasher.hash_table(preimage) - record_id: bytes = record_id_hash.to_prefixed_digest() - - ctx = InvocationContext( - pod_name=pod.label, - pipeline_run_id=run_id, - _pipeline_hash_ch=pipeline_hash_ch, - _record_id_hash_ch=record_id_hash, - _hash_config=pod.pod_config.hash_config, - _track_completion=pod.pod_config.track_completion, - ) - return ctx, record_id_hash, record_id - - -def _build_invocation_context( - *, - pod: "SideEffectFunctionPod", - tag: "TagProtocol", - data: "DataProtocol", - pipeline_hash_ch: "ContentHash", - run_id: str | None, -) -> InvocationContext: - """Convenience wrapper — returns only the ``InvocationContext``. - - Used by ``SideEffectFunctionPodStream.iter_data()`` where the record ID - is not needed. - """ - ctx, _, _ = _build_ctx_and_record_id( - pod=pod, tag=tag, data=data, - pipeline_hash_ch=pipeline_hash_ch, run_id=run_id, - ) - return ctx - - -# --------------------------------------------------------------------------- -# SideEffectFunctionPod -# --------------------------------------------------------------------------- - - -class SideEffectFunctionPod(TraceableBase): - """Function pod that receives an ``InvocationContext`` per row. - - Wraps a callable ``(arg1: T1, ..., ctx: InvocationContext) -> OutputData``. - The ``ctx`` parameter is stripped from schema inference; data fields are - passed as keyword arguments. The pod produces a downstream data stream - like ``FunctionPod``. - - Args: - fn: Callable whose signature includes ``ctx_arg_name`` plus data args. - output_keys: Key(s) mapping the return value(s) to output columns. - A bare string is wrapped in a list. - ctx_arg_name: Name of the context parameter (default ``"ctx"``). - config: Pod-level ``SideEffectPodConfig``. Defaults to - ``SideEffectPodConfig()``. - name: Optional canonical function name override. - version: Version integer in the URI (default ``1``). - label: Optional display label. - data_context: Optional data context override. - - Raises: - ValueError: If ``ctx_arg_name`` is not in ``fn``'s signature. - """ - - def __init__( - self, - fn: Callable, - output_keys: list[str] | str, - ctx_arg_name: str = "ctx", - config: SideEffectPodConfig | None = None, - name: str | None = None, - version: int = 1, - label: str | None = None, - data_context: Any = None, - ) -> None: - super().__init__(label=label, data_context=data_context) - self._fn = fn - self._ctx_arg_name = ctx_arg_name - self._pod_config = config or SideEffectPodConfig() - self._version = version - self._name: str = name if name is not None else getattr(fn, "__name__", "unknown") - self._output_keys: list[str] = ( - [output_keys] if isinstance(output_keys, str) else list(output_keys) - ) - self.tracker_manager: "TrackerManagerProtocol" = DEFAULT_TRACKER_MANAGER - self._is_async = inspect.iscoroutinefunction(fn) - - # Strip ctx for schema inference (raises ValueError if ctx_arg_name missing) - stripped = _strip_ctx_from_fn(fn, ctx_arg_name) - - # Extract schemas from the stripped wrapper - self.input_data_schema, self.output_data_schema = extract_function_schemas( - stripped, output_keys=self._output_keys - ) - - # Pre-compute hashes for URI and result-cache variation data - from orcapod.hashing.hash_utils import get_function_components, get_function_signature - - semantic_hasher = self.data_context.semantic_hasher - self._function_signature_hash = semantic_hasher.hash_object( - get_function_signature(fn) - ).to_string() - self._function_content_hash = semantic_hasher.hash_object( - get_function_components(fn) - ).to_string() - self._output_schema_hash = semantic_hasher.hash_object( - self.output_data_schema - ).to_string() - self._git_hash: str = "" # stable empty string; populated by CI - - # Register Arrow types - self.data_context.type_converter.ensure_types_registered_for_schemas( - self.input_data_schema, - self.output_data_schema, - ) - - # ------------------------------------------------------------------ - # Identity - # ------------------------------------------------------------------ - - @property - def uri(self) -> tuple[str, ...]: - """Canonical URI: ``("side_effect_function", name, schema_hash, "vN", "python_side_effect_function")``.""" - return ( - "side_effect_function", - self.canonical_function_name, - self._output_schema_hash, - f"v{self._version}", - "python_side_effect_function", - ) - - def identity_structure(self) -> Any: - return self.uri - - def pipeline_identity_structure(self) -> Any: - return self.identity_structure() - - # ------------------------------------------------------------------ - # Pod API - # ------------------------------------------------------------------ - - @property - def pod_config(self) -> SideEffectPodConfig: - """Pod-level configuration.""" - return self._pod_config - - @property - def canonical_function_name(self) -> str: - """Human-readable function identifier.""" - return self._name - - def computed_label(self) -> str | None: - """Use the callable's ``__name__`` as the default label.""" - return getattr(self._fn, "__name__", None) - - def argument_symmetry(self, streams: Collection["StreamProtocol"]) -> Any: - """Single ordered input — return as an ordered tuple.""" - return tuple(streams) - - def output_schema( - self, - *streams: "StreamProtocol", - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "tuple[Schema, Schema]": - """Return ``(tag_schema, output_data_schema)`` for the given input streams. - - Args: - *streams: Exactly one input stream. - columns: Optional column config. - all_info: Include all metadata columns. - - Returns: - ``(tag_schema, output_data_schema)`` — tags pass through unchanged. - - Raises: - ValueError: If ``streams`` does not contain exactly one stream. - """ - if len(streams) != 1: - raise ValueError( - f"SideEffectFunctionPod expects exactly 1 input stream; got {len(streams)}." - ) - tag_schema, _ = streams[0].output_schema(columns=columns, all_info=all_info) - return tag_schema, self.output_data_schema - - def process( - self, *streams: "StreamProtocol", label: str | None = None - ) -> "SideEffectFunctionPodStream": - """Invoke the pod on the input stream. - - Records a ``SideEffectFunctionInvocation`` when inside a pipeline - recording block, then returns a ``SideEffectFunctionPodStream``. - - Args: - *streams: Exactly one input stream. - label: Optional display label. - - Returns: - A ``SideEffectFunctionPodStream``. - - Raises: - ValueError: If ``streams`` does not contain exactly one stream. - """ - if len(streams) != 1: - raise ValueError( - f"SideEffectFunctionPod.process() expects exactly 1 stream; got {len(streams)}." - ) - input_stream = streams[0] - self.tracker_manager.record_side_effect_function_pod_invocation( - self, input_stream, label=label - ) - return SideEffectFunctionPodStream(pod=self, input_stream=input_stream, label=label) - - def __call__( - self, *streams: "StreamProtocol", label: str | None = None - ) -> "SideEffectFunctionPodStream": - """Convenience alias for ``process``.""" - return self.process(*streams, label=label) - - # ------------------------------------------------------------------ - # Internal execution helpers - # ------------------------------------------------------------------ - - def _call_with_ctx(self, data: "DataProtocol", ctx: InvocationContext) -> Any: - """Call the user function with data kwargs and InvocationContext. - - Args: - data: Input data row. - ctx: Per-row ``InvocationContext``. - - Returns: - Raw function return value. - """ - kwargs = {self._ctx_arg_name: ctx, **data.as_dict()} - if self._is_async: - return self._call_async_sync(kwargs) - return self._fn(**kwargs) - - def _call_async_sync(self, kwargs: dict[str, Any]) -> Any: - """Run the async user function synchronously. - - Args: - kwargs: Keyword arguments to pass to ``self._fn``. - - Returns: - The coroutine's return value. - """ - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(self._fn(**kwargs)) - - import concurrent.futures - with concurrent.futures.ThreadPoolExecutor(1) as executor: - future = executor.submit(asyncio.run, self._fn(**kwargs)) - return future.result() - - def _build_output_data(self, raw_output: Any) -> "DataProtocol": - """Wrap raw function return value in a ``Data`` object with source info. - - Args: - raw_output: Raw return value from the user function. - - Returns: - A ``Data`` with source info and a new UUID. - """ - from orcapod.core.datagrams import Data - from orcapod.core.data_function import parse_function_outputs - - output_dict = parse_function_outputs(self._output_keys, raw_output) - new_uuid = uuid.UUID(bytes=uuid7().bytes) - source_info = { - k: f"{':'.join(self.uri)}::{new_uuid.hex}::{k}" for k in output_dict - } - return Data( - output_dict, - source_info=source_info, - record_uuid=new_uuid, - python_schema=self.output_data_schema, - data_context=self.data_context, - ) - - # ------------------------------------------------------------------ - # Result cache metadata (mirrors PythonDataFunction) - # ------------------------------------------------------------------ - - def get_function_variation_data(self) -> dict[str, Any]: - """Data defining function variation for ``ResultCache.store()``.""" - return { - "function_name": self.canonical_function_name, - "function_signature_hash": self._function_signature_hash, - "function_content_hash": self._function_content_hash, - "git_hash": self._git_hash, - } - - def get_function_variation_data_schema(self) -> "Schema": - """Schema for ``get_function_variation_data``.""" - from orcapod.types import Schema - return Schema({ - "function_name": str, - "function_signature_hash": str, - "function_content_hash": str, - "git_hash": str, - }) - - def get_execution_data(self) -> dict[str, Any]: - """Data defining execution context for ``ResultCache.store()``.""" - vi = sys.version_info - return { - "executor_type": "none", - "executor_info": {}, - "python_version": f"{vi.major}.{vi.minor}.{vi.micro}", - "extra_info": {}, - } - - def get_execution_data_schema(self) -> "Schema": - """Schema for ``get_execution_data``.""" - from orcapod.types import Schema - return Schema({ - "executor_type": str, - "executor_info": dict[str, str], - "python_version": str, - "extra_info": dict[str, str], - }) - - -# --------------------------------------------------------------------------- -# SideEffectFunctionPodStream — standalone execution (no DB) -# --------------------------------------------------------------------------- - - -class SideEffectFunctionPodStream(StreamBase): - """Lazy stream returned by ``SideEffectFunctionPod.process()`` in standalone mode. - - Iterates the upstream stream, builds a per-row ``InvocationContext``, - calls the user function, and yields ``(tag, output_data)`` pairs. - No invocation log is written in standalone mode (``run_id=None``). - - Args: - pod: The ``SideEffectFunctionPod`` this stream wraps. - input_stream: The upstream stream. - """ - - node_type = "side_effect_function" - - def __init__( - self, - pod: SideEffectFunctionPod, - input_stream: "StreamProtocol", - **kwargs: Any, - ) -> None: - self._pod = pod - self._input_stream = input_stream - super().__init__(**kwargs) - - @property - def producer(self) -> SideEffectFunctionPod: - return self._pod - - @property - def upstreams(self) -> "tuple[StreamProtocol, ...]": - return (self._input_stream,) - - def identity_structure(self) -> Any: - return (self._pod, self._pod.argument_symmetry((self._input_stream,))) - - def pipeline_identity_structure(self) -> Any: - return self.identity_structure() - - def output_schema( - self, - *, - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "tuple[Schema, Schema]": - tag_schema, _ = self._input_stream.output_schema(columns=columns, all_info=all_info) - return tag_schema, self._pod.output_data_schema - - def keys( - self, - *, - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "tuple[tuple[str, ...], tuple[str, ...]]": - tag_schema, data_schema = self.output_schema(columns=columns, all_info=all_info) - return tuple(tag_schema.keys()), tuple(data_schema.keys()) - - def iter_data(self) -> "Iterator[tuple[TagProtocol, DataProtocol]]": - """Iterate the input stream, calling the pod function per row. - - Exceptions from the user function always propagate. - """ - for tag, data in self._input_stream.iter_data(): - ctx = _build_invocation_context( - pod=self._pod, - tag=tag, - data=data, - pipeline_hash_ch=self.pipeline_hash(), - run_id=None, - ) - raw = self._pod._call_with_ctx(data, ctx) - output_data = self._pod._build_output_data(raw) - yield tag, output_data - - def as_table( - self, - *, - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "pa.Table": - """Collect all rows from ``iter_data()`` into an Arrow table.""" - from orcapod.types import ColumnConfig as _ColumnConfig - from orcapod.utils import arrow_utils - - column_config = _ColumnConfig.handle_config(columns, all_info=all_info) - tag_tables = [] - data_tables = [] - for tag, data in self.iter_data(): - tag_tables.append(tag.as_table(columns=column_config)) - data_tables.append(data.as_table(columns=column_config)) - if not tag_tables: - tag_schema, data_schema = self.output_schema(columns=column_config) - tc = self._pod.data_context.type_converter - fields = { - name: pa.array([], type=tc.python_type_to_arrow_type(py_type)) - for name, py_type in {**tag_schema, **data_schema}.items() - } - return pa.table(fields) - return arrow_utils.hstack_tables( - pa.concat_tables(tag_tables), - pa.concat_tables(data_tables), - ) - - -# --------------------------------------------------------------------------- -# SideEffectFunctionNode — blueprint (raises on iter_data) -# --------------------------------------------------------------------------- - - -class SideEffectFunctionNode(StreamBase): - """Lightweight blueprint node for side-effect function pods. - - Used by ``Pipeline`` to represent a ``SideEffectFunctionPod`` invocation - without any DB attachment or execution logic. ``iter_data()`` raises - ``PipelineJobRequiredError`` — use a ``PipelineJob`` to execute. - - Args: - pod: The ``SideEffectFunctionPod`` this node wraps. - input_stream: The upstream stream at compile time. - label: Optional display label. - """ - - node_type = "side_effect_function" - - def __init__( - self, - pod: SideEffectFunctionPod, - input_stream: "StreamProtocol", - label: str | None = None, - ) -> None: - self._pod = pod - self._input_stream = input_stream - super().__init__(label=label) - - @property - def producer(self) -> SideEffectFunctionPod: - return self._pod - - @property - def upstreams(self) -> "tuple[StreamProtocol, ...]": - return (self._input_stream,) - - def identity_structure(self) -> Any: - return (self._pod, self._pod.argument_symmetry((self._input_stream,))) - - def pipeline_identity_structure(self) -> Any: - return self.identity_structure() - - def computed_label(self) -> str | None: - return self._pod.label - - def output_schema( - self, - *, - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "tuple[Schema, Schema]": - tag_schema, _ = self._input_stream.output_schema(columns=columns, all_info=all_info) - return tag_schema, self._pod.output_data_schema - - def keys( - self, - *, - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "tuple[tuple[str, ...], tuple[str, ...]]": - tag_schema, data_schema = self.output_schema(columns=columns, all_info=all_info) - return tuple(tag_schema.keys()), tuple(data_schema.keys()) - - def iter_data(self) -> "Iterator[tuple[TagProtocol, DataProtocol]]": - raise PipelineJobRequiredError( - "SideEffectFunctionNode.iter_data() requires a PipelineJob. " - "Use pod.process(stream).iter_data() for standalone execution, " - "or run via a PipelineJob." - ) - - def as_table( - self, - *, - columns: "ColumnConfig | dict[str, Any] | None" = None, - all_info: bool = False, - ) -> "pa.Table": - raise PipelineJobRequiredError( - "SideEffectFunctionNode.as_table() requires a PipelineJob." - ) - - @property - def node_uri(self) -> tuple[str, ...]: - """Canonical URI identifying this node — same as ``pod.uri``.""" - return self._pod.uri - - def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: - """No-op for blueprint nodes.""" - - -# --------------------------------------------------------------------------- -# SideEffectFunctionJobNode — DB-backed execution -# --------------------------------------------------------------------------- - - -class SideEffectFunctionJobNode(SideEffectFunctionNode): - """DB-backed execution node for side-effect function pods. - - Created at pipeline compile time by ``PipelineJob``. Receives databases - via ``attach_databases()``. On each ``execute()`` call: - - 1. Cache hit check (if ``track_completion=True``). - 2. Build ``InvocationContext`` per row. - 3. Call user function — exceptions always propagate. - 4. Wrap output in ``Data`` and cache in result DB. - 5. Write invocation log row to pipeline DB. - - Args: - pod: The ``SideEffectFunctionPod`` this node wraps. - input_stream: The upstream stream at compile time. - label: Optional display label. - """ - - def __init__( - self, - pod: SideEffectFunctionPod, - input_stream: "StreamProtocol", - label: str | None = None, - ) -> None: - super().__init__(pod=pod, input_stream=input_stream, label=label) - self._pipeline_database: "ArrowDatabaseProtocol | None" = None - self._result_cache: Any = None - self._table_path: tuple[str, ...] | None = None - - def attach_databases( - self, - pipeline_database: "ArrowDatabaseProtocol | None" = None, - result_database: "ArrowDatabaseProtocol | None" = None, - ) -> None: - """Attach pipeline and result databases. - - Sets up the result cache and the invocation log table path. - Called by ``PipelineJob._distribute_databases()``. - - Args: - pipeline_database: Pre-scoped pipeline DB for invocation logging. - result_database: DB for output caching via ``ResultCache``. - """ - from orcapod.core.result_cache import ResultCache - - self._pipeline_database = pipeline_database - self._result_cache = ( - ResultCache(result_database, record_path=self.node_uri) - if result_database is not None - else None - ) - if pipeline_database is not None: - self._table_path = self.node_uri + ( - f"schema:{self.pipeline_hash().to_string()}", - ) - else: - self._table_path = None - - def execute( - self, - input_stream: "StreamProtocol", - *, - observer: "ExecutionObserverProtocol | None" = None, - run_id: str | None = None, - ) -> "list[tuple[TagProtocol, DataProtocol]]": - """Execute side-effect function delivery for all rows in ``input_stream``. - - Args: - input_stream: Stream of ``(tag, data)`` pairs to process. - observer: Optional execution observer (currently unused). - run_id: Pipeline run identifier from the orchestrator. - - Returns: - List of ``(tag, output_data)`` tuples. - """ - from orcapod.core.datagrams import Datagram - - results: list[tuple[TagProtocol, DataProtocol]] = [] - - for tag, data in input_stream.iter_data(): - # 1. Cache hit check - if ( - self._pod.pod_config.track_completion - and self._result_cache is not None - ): - cached = self._result_cache.lookup(data) - if cached is not None: - results.append((tag, cached)) - continue - - # 2. Build InvocationContext + record_id (single preimage computation) - ctx, record_id_hash, record_id = _build_ctx_and_record_id( - pod=self._pod, - tag=tag, - data=data, - pipeline_hash_ch=self.pipeline_hash(), - run_id=run_id, - ) - - # 3. Call user function — always re-raise (no silent row suppression) - try: - raw = self._pod._call_with_ctx(data, ctx) - except Exception as exc: - if self._pod.pod_config.on_error == "log": - logger.warning( - "SideEffectFunctionPod %r failed on row: %s", - self._pod.label, - exc, - exc_info=True, - ) - raise # always re-raise - - # 4. Wrap output and cache it - output_data = self._pod._build_output_data(raw) - if self._result_cache is not None: - var_dg = Datagram( - self._pod.get_function_variation_data(), - python_schema=self._pod.get_function_variation_data_schema(), - data_context=self._pod.data_context, - ) - exec_dg = Datagram( - self._pod.get_execution_data(), - python_schema=self._pod.get_execution_data_schema(), - data_context=self._pod.data_context, - ) - self._result_cache.store(data, output_data, var_dg, exec_dg) - - # 5. Log invocation to pipeline database - if self._pipeline_database is not None and self._table_path is not None: - _write_invocation_row( - pipeline_database=self._pipeline_database, - table_path=self._table_path, - record_id=record_id, - record_id_hash_str=record_id_hash.to_string(), - run_id=run_id, - ) - - results.append((tag, output_data)) - - return results - - async def async_execute( - self, - inputs: "Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]]", - output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", - *, - observer: "ExecutionObserverProtocol | None" = None, - run_id: str | None = None, - ) -> None: - """Async execution with semaphore-bounded concurrency. - - Reads from ``inputs[0]``, dispatches each row as an independent - async task via ``asyncio.TaskGroup``. A semaphore bounds in-flight - tasks at ``pod_config.max_concurrency`` (default 16). Always closes - ``output`` in a ``finally`` block. - - Args: - inputs: Single-element sequence with the input channel. - output: Writable channel for output ``(tag, output_data)`` pairs. - observer: Optional execution observer (currently unused). - run_id: Pipeline run identifier from the orchestrator. - - Raises: - ValueError: If ``inputs`` does not contain exactly one channel. - """ - if len(inputs) != 1: - raise ValueError( - f"SideEffectFunctionJobNode.async_execute expects exactly 1 " - f"input channel; got {len(inputs)}." - ) - - max_concurrency = self._pod.pod_config.max_concurrency - sem = asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None - - try: - async def process_one(tag: "TagProtocol", data: "DataProtocol") -> None: - try: - from orcapod.core.datagrams import Datagram - - # Cache hit check - if ( - self._pod.pod_config.track_completion - and self._result_cache is not None - ): - cached = self._result_cache.lookup(data) - if cached is not None: - await output.send((tag, cached)) - return - - ctx, record_id_hash, record_id = _build_ctx_and_record_id( - pod=self._pod, - tag=tag, - data=data, - pipeline_hash_ch=self.pipeline_hash(), - run_id=run_id, - ) - - try: - raw = self._pod._call_with_ctx(data, ctx) - except Exception as exc: - if self._pod.pod_config.on_error == "log": - logger.warning( - "SideEffectFunctionPod %r async failed: %s", - self._pod.label, - exc, - exc_info=True, - ) - raise - - output_data = self._pod._build_output_data(raw) - - if self._result_cache is not None: - var_dg = Datagram( - self._pod.get_function_variation_data(), - python_schema=self._pod.get_function_variation_data_schema(), - data_context=self._pod.data_context, - ) - exec_dg = Datagram( - self._pod.get_execution_data(), - python_schema=self._pod.get_execution_data_schema(), - data_context=self._pod.data_context, - ) - self._result_cache.store(data, output_data, var_dg, exec_dg) - - if self._pipeline_database is not None and self._table_path is not None: - _write_invocation_row( - pipeline_database=self._pipeline_database, - table_path=self._table_path, - record_id=record_id, - record_id_hash_str=record_id_hash.to_string(), - run_id=run_id, - ) - - await output.send((tag, output_data)) - finally: - if sem is not None: - sem.release() - - async with asyncio.TaskGroup() as tg: - async for tag, data in inputs[0]: - if sem is not None: - await sem.acquire() - tg.create_task(process_one(tag, data)) - finally: - await output.close() - - def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: - """No-op for this node type.""" - - -# --------------------------------------------------------------------------- -# Decorator -# --------------------------------------------------------------------------- - - -def side_effect_function_pod( - fn: Callable | None = None, - *, - output_keys: list[str] | str, - ctx_arg_name: str = "ctx", - config: SideEffectPodConfig | None = None, - name: str | None = None, - version: int = 1, -) -> "SideEffectFunctionPod | Callable": - """Decorator wrapping a callable as a ``SideEffectFunctionPod``. - - Parameterised usage only — ``output_keys`` is always required: - - .. code-block:: python - - @side_effect_function_pod(output_keys=["artifact_path"]) - def write_artifact(value: int, ctx: InvocationContext) -> Path: - path = Path(f"out/{ctx.invocation_hash}.bin") - path.write_bytes(value.to_bytes(4, "big")) - return path - - The decorated name is replaced with a ``SideEffectFunctionPod`` instance. - The pod is callable (via ``__call__`` → ``process``). - - Args: - fn: Internal — not for direct caller use. - output_keys: Key(s) mapping the return value(s) to output columns. - ctx_arg_name: Name of the context parameter (default ``"ctx"``). - config: Optional ``SideEffectPodConfig``. - name: Optional canonical function name override. - version: Version integer for the URI (default ``1``). - - Returns: - A ``SideEffectFunctionPod`` (or a one-argument decorator). - """ - def _wrap(f: Callable) -> SideEffectFunctionPod: - return SideEffectFunctionPod( - f, - output_keys=output_keys, - ctx_arg_name=ctx_arg_name, - config=config, - name=name, - version=version, - ) - - if fn is not None: - return _wrap(fn) - return _wrap -``` - -- [ ] **Step 5: Run tests to confirm SF-01, SF-02, SF-03, SF-10 pass** - -```bash -uv run pytest tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectFunctionPodSchema -v -``` - -Expected: all 4 tests pass. - -- [ ] **Step 6: Run full test suite to confirm no regressions** - -```bash -uv run pytest tests/ -x -q --tb=short 2>&1 | tail -10 -``` - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/side_effect_function/ tests/test_core/side_effect_function/ -git commit -m "feat(side_effect_function): scaffold module with SideEffectFunctionPod + all node classes" -``` - ---- - -### Task 3: Invocation Type + Protocol + Tracker Plumbing (SF-04, SF-05) - -Add `SideEffectFunctionInvocation`, `SideEffectFunctionPodProtocol`, and `record_side_effect_function_pod_invocation` to tracker and pipeline base. Then test standalone stream execution. - -**Files:** -- Modify: `src/orcapod/pipeline/pod_invocation.py` -- Create: `src/orcapod/protocols/core_protocols/side_effect_function_pod.py` -- Modify: `src/orcapod/protocols/core_protocols/__init__.py` -- Modify: `src/orcapod/protocols/core_protocols/trackers.py` -- Modify: `src/orcapod/core/tracker.py` -- Modify: `src/orcapod/pipeline/base.py` -- Modify: `tests/test_core/side_effect_function/test_side_effect_function_pod.py` (SF-04, SF-05) - -- [ ] **Step 1: Write failing tests SF-04 and SF-05** - -Add to `tests/test_core/side_effect_function/test_side_effect_function_pod.py`: - -```python -class TestSideEffectFunctionPodStreamStandalone: - """SF-04, SF-05: standalone execution via SideEffectFunctionPodStream.""" - - def test_sf04_iter_data_returns_correct_output(self): - """SF-04: iter_data() returns correct (tag, output_data) per row.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod - - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"v{value}" - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - stream = _make_stream(3) - rows = list(pod.process(stream).iter_data()) - - assert len(rows) == 3 - for i, (tag, data) in enumerate(rows): - assert data.as_dict()["result"] == f"v{i}" - # Tags pass through unchanged - assert rows[0][0].as_dict()["id"] == 0 - assert rows[1][0].as_dict()["id"] == 1 - - def test_sf05_invocation_context_fields_standalone(self): - """SF-05: InvocationContext has pod_name, non-empty hash, pipeline_run_id=None.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod - - received_ctx: list[InvocationContext] = [] - - def my_fn(value: int, ctx: InvocationContext) -> str: - received_ctx.append(ctx) - return str(value) - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - stream = _make_stream(1) - list(pod.process(stream).iter_data()) - - assert len(received_ctx) == 1 - ctx = received_ctx[0] - assert ctx.pod_name == pod.label - assert isinstance(ctx.invocation_hash, str) - assert len(ctx.invocation_hash) > 0 - assert "::" in ctx.invocation_hash - assert ctx.pipeline_run_id is None # standalone: no run_id -``` - -- [ ] **Step 2: Run tests to confirm they fail (tracker method missing)** - -```bash -uv run pytest tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectFunctionPodStreamStandalone -x -q 2>&1 | head -20 -``` - -Expected: `AttributeError: 'BasicTrackerManager' object has no attribute 'record_side_effect_function_pod_invocation'` — called from `SideEffectFunctionPod.process()`. - -- [ ] **Step 3: Add `SideEffectFunctionInvocation` to `pod_invocation.py`** - -Add at the end of `src/orcapod/pipeline/pod_invocation.py`: - -```python -class SideEffectFunctionInvocation(PodInvocation): - """Invocation of a side-effect function pod against exactly one input stream. - - Args: - pod: A ``SideEffectFunctionPod`` instance. - input_streams: Tuple with exactly one stream. - label: Optional display label. - - Raises: - ValueError: If ``input_streams`` does not contain exactly one element. - """ - - def __init__( - self, - pod: Any, - input_streams: "tuple[StreamProtocol, ...]", - label: str | None = None, - ) -> None: - if len(input_streams) != 1: - raise ValueError( - f"SideEffectFunctionInvocation requires exactly 1 input stream; " - f"got {len(input_streams)}." - ) - super().__init__(pod=pod, input_streams=input_streams, label=label) -``` - -- [ ] **Step 4: Create `SideEffectFunctionPodProtocol`** - -Create `src/orcapod/protocols/core_protocols/side_effect_function_pod.py`: - -```python -"""Protocol for SideEffectFunctionPod.""" -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable - -from orcapod.protocols.hashing_protocols import PipelineElementProtocol - -if TYPE_CHECKING: - from orcapod.side_effects import SideEffectPodConfig - from orcapod.protocols.core_protocols.streams import StreamProtocol - from orcapod.types import Schema - - -@runtime_checkable -class SideEffectFunctionPodProtocol(PipelineElementProtocol, Protocol): - """Protocol for side-effect function pods. - - Hybrid of ``FunctionPodProtocol`` and ``SideEffectPodProtocol``: - receives per-row ``InvocationContext``, produces a downstream data stream. - """ - - @property - def pod_config(self) -> "SideEffectPodConfig": ... - - @property - def input_data_schema(self) -> "Schema": ... - - @property - def output_data_schema(self) -> "Schema": ... - - def process(self, *streams: "StreamProtocol", label: str | None = None) -> Any: ... - - def output_schema( - self, - *streams: "StreamProtocol", - columns: Any = None, - all_info: bool = False, - ) -> "tuple[Schema, Schema]": ... - - def argument_symmetry(self, streams: Any) -> Any: ... -``` - -- [ ] **Step 5: Export `SideEffectFunctionPodProtocol` from `core_protocols/__init__.py`** - -Add to `src/orcapod/protocols/core_protocols/__init__.py`: - -```python -from .side_effect_function_pod import SideEffectFunctionPodProtocol -``` - -Add `"SideEffectFunctionPodProtocol"` to `__all__`. - -- [ ] **Step 6: Add `record_side_effect_function_pod_invocation` to both tracker protocols** - -In `src/orcapod/protocols/core_protocols/trackers.py`, add to `TrackerProtocol`: - -```python -def record_side_effect_function_pod_invocation( - self, - pod: Any, - input_stream: StreamProtocol, - label: str | None = None, -) -> None: - """Record a side-effect function pod invocation in the computational graph. - - Args: - pod: The ``SideEffectFunctionPod`` being invoked. - input_stream: The upstream stream. - label: Optional display label. - """ - ... -``` - -Add same method to `TrackerManagerProtocol`: - -```python -def record_side_effect_function_pod_invocation( - self, - pod: Any, - input_stream: StreamProtocol, - label: str | None = None, -) -> None: - """Record a side-effect function pod invocation in all active trackers. - - Args: - pod: The ``SideEffectFunctionPod`` to record. - input_stream: The upstream stream. - label: Optional display label. - """ - ... -``` - -- [ ] **Step 7: Add `record_side_effect_function_pod_invocation` to `BasicTrackerManager` in `tracker.py`** - -Add to `BasicTrackerManager` in `src/orcapod/core/tracker.py`: - -```python -def record_side_effect_function_pod_invocation( - self, - pod: Any, - input_stream: cp.StreamProtocol, - label: str | None = None, -) -> None: - """Record a side-effect function pod invocation in all active trackers.""" - for tracker in self.get_active_trackers(): - tracker.record_side_effect_function_pod_invocation(pod, input_stream, label=label) -``` - -- [ ] **Step 8: Wire `AbstractPipelineBase` — add recorder, abstract property, compile + to_invocations** - -In `src/orcapod/pipeline/base.py`: - -**8a. Add imports:** - -```python -from orcapod.pipeline.pod_invocation import ( - FunctionInvocation, - OperatorInvocation, - PodInvocation, - SideEffectFunctionInvocation, # NEW - SideEffectInvocation, -) -``` - -Also add a lazy import for `SideEffectFunctionNode` inside methods (to avoid circular imports). - -**8b. Add abstract property after `side_effect_node_class`:** - -```python -@property -@abstractmethod -def side_effect_function_node_class(self) -> type: - """Node class for side-effect-function pod invocations.""" - ... -``` - -**8c. Add recording method (concrete, like `record_side_effect_pod_invocation`):** - -```python -def record_side_effect_function_pod_invocation( - self, - pod: Any, - input_stream: "cp.StreamProtocol", - label: str | None = None, -) -> None: - """Record a side-effect function pod invocation into the graph. - - Args: - pod: The ``SideEffectFunctionPod`` being invoked. - input_stream: The upstream stream. - label: Optional display label for the resulting compiled node. - """ - self._record_invocation( - SideEffectFunctionInvocation(pod=pod, input_streams=(input_stream,), label=label) - ) -``` - -**8d. Update `compile()` topological pass** — add before the `else` (OperatorInvocation) branch: - -```python -elif isinstance(inv, SideEffectFunctionInvocation): - node_map[key] = self.side_effect_function_node_class( - pod=inv.pod, - input_stream=upstream_nodes[0], - label=inv.label, - ) -``` - -**8e. Update `to_invocations()`** — add after the `SideEffectNode` branch and before the `else` branch: - -```python -# Import at top of the method body or at module level: -from orcapod.core.side_effect_function import SideEffectFunctionNode as _SEFNode - -# Then in the loop: -elif isinstance(node, _SEFNode): - inv_by_node_hash[node_hash] = SideEffectFunctionInvocation( - pod=node._pod, - input_streams=(node.upstreams[0],), - label=node._label, - ) -``` - -To avoid circular imports, use a local import inside the method: - -```python -def to_invocations(self) -> InvocationGraph: - ... - from orcapod.core.side_effect_function.side_effect_function_pod import SideEffectFunctionNode as _SEFNode - ... - elif isinstance(node, _SEFNode): - inv_by_node_hash[node_hash] = SideEffectFunctionInvocation(...) -``` - -- [ ] **Step 9: Run SF-04 and SF-05 to confirm they pass** - -```bash -uv run pytest tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectFunctionPodStreamStandalone -v -``` - -Expected: both tests pass. - -- [ ] **Step 10: Run full test suite** - -```bash -uv run pytest tests/ -x -q --tb=short 2>&1 | tail -15 -``` - -- [ ] **Step 11: Commit** - -```bash -git add \ - src/orcapod/pipeline/pod_invocation.py \ - src/orcapod/protocols/core_protocols/side_effect_function_pod.py \ - src/orcapod/protocols/core_protocols/__init__.py \ - src/orcapod/protocols/core_protocols/trackers.py \ - src/orcapod/core/tracker.py \ - src/orcapod/pipeline/base.py \ - tests/test_core/side_effect_function/test_side_effect_function_pod.py -git commit -m "feat(side_effect_function): invocation type, protocol, tracker plumbing, pipeline base wiring" -``` - ---- - -### Task 4: Node Protocols + Orchestrators + `graph.py`/`job.py` Class Attributes - -Add `SideEffectFunctionNodeProtocol` + `is_side_effect_function_node`, update both orchestrators and Pipeline/PipelineJob class attributes. - -**Files:** -- Modify: `src/orcapod/protocols/node_protocols.py` -- Modify: `src/orcapod/pipeline/sync_orchestrator.py` -- Modify: `src/orcapod/pipeline/async_orchestrator.py` -- Modify: `src/orcapod/pipeline/graph.py` -- Modify: `src/orcapod/pipeline/job.py` - -- [ ] **Step 1: Add `SideEffectFunctionNodeProtocol` and `is_side_effect_function_node` to `node_protocols.py`** - -```python -# In src/orcapod/protocols/node_protocols.py, add after SideEffectNodeProtocol: - -@runtime_checkable -class SideEffectFunctionNodeProtocol(Protocol): - """Protocol for side-effect-function nodes in orchestrated execution. - - Combines function-pod output production with per-row ``InvocationContext`` - injection and optional DB-backed caching + invocation logging. - """ - - node_type: str - - def execute( - self, - input_stream: "StreamProtocol", - *, - observer: "ExecutionObserverProtocol | None" = None, - run_id: str | None = None, - ) -> "list[tuple[TagProtocol, DataProtocol]]": ... - - async def async_execute( - self, - inputs: "Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]]", - output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", - *, - observer: "ExecutionObserverProtocol | None" = None, - run_id: str | None = None, - ) -> None: ... - - def attach_databases( - self, - pipeline_database: "ArrowDatabaseProtocol | None" = None, - result_database: "ArrowDatabaseProtocol | None" = None, - ) -> None: ... - - -def is_side_effect_function_node(node: "GraphNode") -> TypeGuard[SideEffectFunctionNodeProtocol]: - """Check if a node is a side-effect-function node.""" - return node.node_type == "side_effect_function" -``` - -- [ ] **Step 2: Update `sync_orchestrator.py`** - -Add import: -```python -from orcapod.protocols.node_protocols import ( - is_function_node, - is_operator_node, - is_side_effect_function_node, # NEW - is_side_effect_node, - is_source_node, -) -``` - -Add branch in `run()` after `elif is_side_effect_node(node):` and before `else: raise TypeError`: - -```python -elif is_side_effect_function_node(node): - upstream_buf = self._gather_upstream(node, graph, buffers) - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream(upstream_buf, upstream_node) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) -``` - -- [ ] **Step 3: Update `async_orchestrator.py`** - -Add same import change. Add branch after `elif is_side_effect_node(node):` and before `else: raise TypeError`: - -```python -elif is_side_effect_function_node(node): - predecessors = in_edges.get(node, []) - if not predecessors: - raise ValueError("SideEffectFunctionNode expects exactly 1 upstream, got 0") - input_reader = edge_readers[(predecessors[0], node)] - tg.create_task( - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ) - ) -``` - -- [ ] **Step 4: Update `graph.py` — add `side_effect_function_node_class` class attribute** - -In `src/orcapod/pipeline/graph.py`: - -Add import: -```python -from orcapod.core.side_effect_function.side_effect_function_pod import SideEffectFunctionNode -``` - -Add to `Pipeline` class: -```python -side_effect_function_node_class = SideEffectFunctionNode -``` - -- [ ] **Step 5: Update `job.py` — add class attr, `_distribute_databases`, `as_pipeline`** - -In `src/orcapod/pipeline/job.py`: - -Add import: -```python -from orcapod.core.side_effect_function.side_effect_function_pod import ( - SideEffectFunctionJobNode, - SideEffectFunctionNode, -) -``` - -Add to `PipelineJob` class attributes: -```python -side_effect_function_node_class = SideEffectFunctionJobNode -``` - -In `_distribute_databases()`, add after the `elif isinstance(node, SideEffectJobNode):` branch: - -```python -elif isinstance(node, SideEffectFunctionJobNode): - node.attach_databases( - pipeline_database=pipeline_db, - result_database=result_db, - ) -``` - -In `as_pipeline()`, add after the `elif isinstance(job_node, SideEffectJobNode):` branch and before the final `else: node_map[node_hash] = job_node.as_node()`: - -```python -elif isinstance(job_node, SideEffectFunctionJobNode): - upstream_bp_hash = job_id_to_bp_hash[id(job_node._input_stream)] - node_map[node_hash] = SideEffectFunctionNode( - pod=job_node._pod, - input_stream=node_map[upstream_bp_hash], - label=job_node._label, - ) -``` - -- [ ] **Step 6: Run existing tests to confirm no regressions** - -```bash -uv run pytest tests/ -x -q --tb=short 2>&1 | tail -15 -``` - -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add \ - src/orcapod/protocols/node_protocols.py \ - src/orcapod/pipeline/sync_orchestrator.py \ - src/orcapod/pipeline/async_orchestrator.py \ - src/orcapod/pipeline/graph.py \ - src/orcapod/pipeline/job.py -git commit -m "feat(side_effect_function): node protocol + orchestrator dispatch + pipeline class attrs" -``` - ---- - -### Task 5: DB-Backed Execution Tests (SF-06–SF-09) + Decorator + Re-exports + Pipeline Integration (SF-11–SF-13) - -Add remaining tests and wire the decorator and public API. All implementation code already exists from Task 2 — these tests verify the already-implemented `SideEffectFunctionJobNode.execute()`, `async_execute()`, and `side_effect_function_pod` decorator. - -**Files:** -- Modify: `tests/test_core/side_effect_function/test_side_effect_function_pod.py` (SF-06–SF-13) -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Add tests SF-06, SF-07, SF-08, SF-09 to the test file** - -```python -class TestSideEffectFunctionJobNode: - """SF-06, SF-07, SF-08, SF-09: DB-backed sync execution.""" - - def test_sf06_output_cached_after_first_run(self): - """SF-06: Output cached; second run returns cached result without re-calling fn.""" - from orcapod.core.side_effect_function import SideEffectFunctionPod, SideEffectFunctionJobNode - - call_count = 0 - - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"r{value}" - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - stream = _make_stream(2) - pipeline_db = _make_in_memory_db() - result_db = _make_in_memory_db() - - node1 = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - results1 = node1.execute(stream) - assert len(results1) == 2 - assert call_count == 2 - - # Second run — same pod, same data, same DBs — fn must NOT be called again - node2 = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - results2 = node2.execute(stream) - assert len(results2) == 2 - assert call_count == 2 # NOT incremented — cache hit - - # Both runs produce equal result values - for (_, d1), (_, d2) in zip(results1, results2): - assert d1.as_dict()["result"] == d2.as_dict()["result"] - - def test_sf07_invocation_log_written_on_first_run(self): - """SF-07: Invocation log row written to pipeline_database on first run.""" - import polars as pl - from orcapod.core.side_effect_function import SideEffectFunctionPod, SideEffectFunctionJobNode - - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"r{value}" - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - stream = _make_stream(3) - pipeline_db = _make_in_memory_db() - result_db = _make_in_memory_db() - node = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - results = node.execute(stream) - - assert len(results) == 3 - - # Invocation log has 3 rows - table_path = node._table_path - records = pipeline_db.get_all_records(table_path) - assert records is not None - df = pl.from_arrow(records) - assert len(df) == 3 - assert "record_id_hash" in df.columns - assert "executed_at" in df.columns - - def test_sf08_track_completion_false_always_reruns(self): - """SF-08: track_completion=False — fn called every run; invocation logged each time.""" - from orcapod.side_effects import SideEffectPodConfig - from orcapod.core.side_effect_function import SideEffectFunctionPod, SideEffectFunctionJobNode - - call_count = 0 - - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"r{value}" - - cfg = SideEffectPodConfig(track_completion=False) - pod = SideEffectFunctionPod(my_fn, output_keys=["result"], config=cfg) - stream = _make_stream(2) - pipeline_db = _make_in_memory_db() - result_db = _make_in_memory_db() - - node1 = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - node1.execute(stream) - assert call_count == 2 - - node2 = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - node2.execute(stream) - assert call_count == 4 # called again — track_completion=False - - def test_sf09_on_error_log_reraises(self): - """SF-09: on_error='log' — exception logged then always re-raised.""" - from orcapod.side_effects import SideEffectPodConfig - from orcapod.core.side_effect_function import SideEffectFunctionPod, SideEffectFunctionJobNode - - def my_fn(value: int, ctx: InvocationContext) -> str: - raise RuntimeError("test error") - - cfg = SideEffectPodConfig(on_error="log") - pod = SideEffectFunctionPod(my_fn, output_keys=["result"], config=cfg) - stream = _make_stream(1) - pipeline_db = _make_in_memory_db() - result_db = _make_in_memory_db() - node = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - - # Must propagate — no silent row suppression - with pytest.raises(RuntimeError, match="test error"): - node.execute(stream) -``` - -- [ ] **Step 2: Run SF-06–SF-09 tests** - -```bash -uv run pytest tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectFunctionJobNode -v -``` - -Expected: all 4 tests pass. - -- [ ] **Step 3: Add tests SF-11, SF-12, SF-13** - -```python -class TestSideEffectFunctionPodDecorator: - """SF-11: @side_effect_function_pod decorator.""" - - def test_sf11_decorator_creates_correct_pod(self): - """SF-11: Decorator creates a SideEffectFunctionPod with correct URI.""" - from orcapod.core.side_effect_function import ( - SideEffectFunctionPod, - side_effect_function_pod, - ) - - @side_effect_function_pod(output_keys=["result"]) - def my_fn(value: int, ctx: InvocationContext) -> str: - return str(value) - - assert isinstance(my_fn, SideEffectFunctionPod) - assert my_fn.uri[0] == "side_effect_function" - assert my_fn.canonical_function_name == "my_fn" - - def test_sf11_decorator_accessible_from_public_api(self): - """SF-11: Decorator and pod class accessible from orcapod top-level.""" - import orcapod - assert hasattr(orcapod, "side_effect_function_pod") - assert hasattr(orcapod, "SideEffectFunctionPod") - - -class TestSideEffectFunctionPodPipelineIntegration: - """SF-12: Full pipeline compilation and execution.""" - - def test_sf12_pipeline_compilation_and_execution(self): - """SF-12: SideEffectFunctionJobNode compiled, fn called, invocation logged.""" - import polars as pl - from orcapod.pipeline.job import PipelineJob - from orcapod.core.side_effect_function import SideEffectFunctionPod - from orcapod.core.sources.dict_source import DictSource - - received_ctx: list[InvocationContext] = [] - - def transform(value: int, ctx: InvocationContext) -> str: - received_ctx.append(ctx) - return f"result_{value}" - - pod = SideEffectFunctionPod(transform, output_keys=["result"]) - db = _make_in_memory_db() - - with PipelineJob(name="test_sef", store=db) as job: - source = DictSource( - [{"id": 0, "value": 10}, {"id": 1, "value": 20}], - tag_columns=["id"], - ) - pod.process(source) - - job.run() - - assert len(received_ctx) == 2 # fn called once per row - - # Find the side_effect_function node in the compiled graph - sef_nodes = [ - n for n in job.dag.nodes() - if getattr(n, "node_type", None) == "side_effect_function" - ] - assert len(sef_nodes) == 1 - node = sef_nodes[0] - - # Invocation log written to pipeline DB - pipeline_db = db.at("test_sef") - table_path = node._table_path - records = pipeline_db.get_all_records(table_path) - assert records is not None - df = pl.from_arrow(records) - assert len(df) == 2 - assert "record_id_hash" in df.columns - - def test_sf12_second_pipeline_run_uses_cache(self): - """SF-12: Second pipeline run uses cached output; fn not called again.""" - from orcapod.pipeline.job import PipelineJob - from orcapod.core.side_effect_function import SideEffectFunctionPod - from orcapod.core.sources.dict_source import DictSource - - call_count = 0 - - def transform(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"r{value}" - - pod = SideEffectFunctionPod(transform, output_keys=["result"]) - db = _make_in_memory_db() - source_data = [{"id": 0, "value": 10}, {"id": 1, "value": 20}] - - with PipelineJob(name="test_sef_cache", store=db) as job1: - source1 = DictSource(source_data, tag_columns=["id"]) - pod.process(source1) - job1.run() - assert call_count == 2 - - with PipelineJob(name="test_sef_cache", store=db) as job2: - source2 = DictSource(source_data, tag_columns=["id"]) - pod.process(source2) - job2.run() - assert call_count == 2 # NOT called again — cache hit - - -class TestSideEffectFunctionJobNodeAsync: - """SF-13: async_execute produces same output as sync path.""" - - def test_sf13_async_execute_basic(self): - """SF-13: async_execute processes all rows, writes cache + log, returns correct output.""" - import asyncio - import polars as pl - from orcapod.core.side_effect_function import SideEffectFunctionPod, SideEffectFunctionJobNode - from orcapod.channels import Channel - - call_count = 0 - - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"async_{value}" - - pod = SideEffectFunctionPod(my_fn, output_keys=["result"]) - stream = _make_stream(3) - pipeline_db = _make_in_memory_db() - result_db = _make_in_memory_db() - node = SideEffectFunctionJobNode(pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=pipeline_db, result_database=result_db) - - async def _run(): - ch_in = Channel(buffer_size=10) - ch_out = Channel(buffer_size=10) - - async def feed(): - for tag, data in stream.iter_data(): - await ch_in.writer.send((tag, data)) - await ch_in.writer.close() - - await asyncio.gather( - feed(), - node.async_execute([ch_in.reader], ch_out.writer, run_id="test-async"), - ) - return await ch_out.reader.collect() - - results = asyncio.run(_run()) - assert len(results) == 3 - assert call_count == 3 - - # Invocation log written - records = pipeline_db.get_all_records(node._table_path) - df = pl.from_arrow(records) - assert len(df) == 3 - - # Output values are correct - result_values = {data.as_dict()["result"] for _, data in results} - assert result_values == {f"async_{i}" for i in range(3)} -``` - -- [ ] **Step 4: Add re-exports to `src/orcapod/__init__.py`** - -```python -# Add after the existing side_effects imports: -from orcapod.core.side_effect_function import ( - SideEffectFunctionPod, - SideEffectFunctionNode, - SideEffectFunctionJobNode, - side_effect_function_pod, -) - -# Add to __all__: -"SideEffectFunctionPod", -"SideEffectFunctionNode", -"SideEffectFunctionJobNode", -"side_effect_function_pod", -``` - -- [ ] **Step 5: Run all SF tests** - -```bash -uv run pytest tests/test_core/side_effect_function/ -v -``` - -Expected: all 13 test cases (SF-01 through SF-13) pass. - -- [ ] **Step 6: Run full test suite** - -```bash -uv run pytest tests/ -q --tb=short 2>&1 | tail -20 -``` - -Expected: all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add \ - tests/test_core/side_effect_function/test_side_effect_function_pod.py \ - src/orcapod/__init__.py -git commit -m "feat(side_effect_function): SF-06–SF-13 tests + public API re-exports" -``` - ---- - -## Self-Review - -### 1. Spec Coverage - -| Spec requirement | Task | -|-----------------|------| -| `SideEffectPod` breaking change: `ctx_arg_name` + `**kwargs` call style | Task 1 | -| `SideEffectPod` decorators gain `ctx_arg_name` | Task 1 | -| `identity_structure()` includes `ctx_arg_name` | Task 1 | -| `_strip_ctx_from_fn` helper | Task 2 | -| `SideEffectFunctionPod` constructor + schema extraction | Task 2 | -| URI: `("side_effect_function", name, schema_hash, "vN", "python_side_effect_function")` | Task 2 | -| `_call_with_ctx` + `_build_output_data` | Task 2 | -| `get_function_variation_data` + `get_execution_data` | Task 2 | -| `SideEffectFunctionPodStream.iter_data()` (standalone) | Task 2 | -| `_build_invocation_context` / `_build_ctx_and_record_id` | Task 2 | -| `SideEffectFunctionNode` (blueprint, raises on iter_data) | Task 2 | -| `SideEffectFunctionJobNode.attach_databases()` | Task 2 | -| `SideEffectFunctionJobNode.execute()` with cache + invocation log | Task 2 | -| `SideEffectFunctionJobNode.async_execute()` | Task 2 | -| `SideEffectFunctionInvocation` | Task 3 | -| `SideEffectFunctionPodProtocol` | Task 3 | -| `record_side_effect_function_pod_invocation` in tracker chain | Task 3 | -| `AbstractPipelineBase.compile()` handles `SideEffectFunctionInvocation` | Task 3 | -| `AbstractPipelineBase.to_invocations()` handles `SideEffectFunctionNode` | Task 3 | -| `SideEffectFunctionNodeProtocol` + `is_side_effect_function_node` | Task 4 | -| Sync orchestrator `elif is_side_effect_function_node(node):` branch | Task 4 | -| Async orchestrator `elif is_side_effect_function_node(node):` branch | Task 4 | -| `Pipeline.side_effect_function_node_class = SideEffectFunctionNode` | Task 4 | -| `PipelineJob.side_effect_function_node_class = SideEffectFunctionJobNode` | Task 4 | -| `PipelineJob._distribute_databases()` handles `SideEffectFunctionJobNode` | Task 4 | -| `PipelineJob.as_pipeline()` handles `SideEffectFunctionJobNode` | Task 4 | -| `@side_effect_function_pod` decorator | Task 2 (impl) / Task 5 (tests) | -| Re-exports from `orcapod.__init__` | Task 5 | -| Test suite SF-01–SF-13 + SEP-UPDATE + SEP-CTX-NAME | Tasks 1–5 | - -All spec requirements covered. ✓ - -### 2. Placeholder Scan - -No "TBD", "TODO", or incomplete code blocks. ✓ - -### 3. Type Consistency - -- `SideEffectFunctionNode(pod=..., input_stream=..., label=...)` — used consistently in compile(), as_pipeline(), and to_invocations(). -- `SideEffectFunctionJobNode(pod=..., input_stream=..., label=...)` — same constructor, used in compile() for PipelineJob. -- `_build_ctx_and_record_id()` returns `(InvocationContext, ContentHash, bytes)` — used identically in `execute()` and `async_execute()`. -- `node_type = "side_effect_function"` — used in `SideEffectFunctionNode` (and inherited by `SideEffectFunctionJobNode`), `SideEffectFunctionPodStream`, and checked by `is_side_effect_function_node()`. ✓ diff --git a/superpowers/plans/2026-07-16-itl-532-unified-function-pod.md b/superpowers/plans/2026-07-16-itl-532-unified-function-pod.md deleted file mode 100644 index f562433e9..000000000 --- a/superpowers/plans/2026-07-16-itl-532-unified-function-pod.md +++ /dev/null @@ -1,1820 +0,0 @@ -# ITL-532 Unified FunctionPod Design — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Eliminate the separate `SideEffectFunctionPod` class hierarchy by making `InvocationContext` injection an optional feature of `FunctionPod` controlled by `ctx_arg_name`. Drop PR #230, start fresh. - -**Architecture:** When `ctx_arg_name` is set on a `FunctionPod`, the URI is prefixed with `"side_effect_function"`, `node_type` returns `"side_effect_function"`, and `process_data()` builds an `InvocationContext` per row and injects it as an extra kwarg. `FunctionJobNode.execute()` and `async_execute()` gain `run_id: str | None = None` threaded to ctx building. The entire `src/orcapod/core/side_effect_function/` module is deleted; the `side_effect_function_pod` decorator becomes a thin alias in `function_pod.py`. - -**Tech Stack:** Python 3.11+, PyArrow, orcapod core (TraceableBase, CachedFunctionPod, FunctionJobNode, InvocationContext), uuid_utils, uv, pytest. - ---- - -## File Map - -| File | Change | -|------|--------| -| `src/orcapod/core/function_pod.py` | Modify — ctx_arg_name support, `from_fn` classmethod, `side_effect_function_pod` decorator | -| `src/orcapod/core/cached_function_pod.py` | Modify — thread `run_id` through `process_data` / `async_process_data` | -| `src/orcapod/core/nodes/function_node.py` | Modify — `node_type` property, `run_id` on `execute` / `async_execute` | -| `src/orcapod/protocols/node_protocols.py` | Modify — remove `SideEffectFunctionNodeProtocol`, add `run_id` to `FunctionNodeProtocol` | -| `src/orcapod/pipeline/sync_orchestrator.py` | Modify — collapse SEF dispatch into function_node branch | -| `src/orcapod/pipeline/async_orchestrator.py` | Modify — same | -| `src/orcapod/pipeline/base.py` | Modify — remove SEF abstract property, recorder, compile branch | -| `src/orcapod/pipeline/pod_invocation.py` | Modify — remove `SideEffectFunctionInvocation` | -| `src/orcapod/pipeline/job.py` | Modify — remove SEF class attr, distribute, as_pipeline branches | -| `src/orcapod/pipeline/graph.py` | Modify — remove SEF import, class attr, save() branch | -| `src/orcapod/core/tracker.py` | Modify — remove `record_side_effect_function_pod_invocation` | -| `src/orcapod/protocols/core_protocols/trackers.py` | Modify — remove same method from protocols | -| `src/orcapod/protocols/core_protocols/side_effect_function_pod.py` | **Delete** | -| `src/orcapod/protocols/core_protocols/__init__.py` | Modify — remove SEF protocol import | -| `src/orcapod/__init__.py` | Modify — remove SEF class exports, keep `side_effect_function_pod` | -| `src/orcapod/core/side_effect_function/` | **Delete entire directory** | -| `tests/test_core/side_effect_function/test_side_effect_function_pod.py` | Modify — use `FunctionPod.from_fn(..., ctx_arg_name="ctx")` | - ---- - -### Task 1: Add ctx support to `_FunctionPodBase` and `FunctionPod` - -This is the heart of the change. All ctx-aware logic from `SideEffectFunctionPod` moves into `_FunctionPodBase`/`FunctionPod`. - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` - -- [ ] **Step 1: Add `_strip_ctx_from_fn` helper and import `inspect` at module level** - -In `function_pod.py`, after the existing imports, add: - -```python -import inspect - -def _strip_ctx_from_fn(fn: Callable, ctx_arg_name: str) -> Callable: - """Return a wrapper of ``fn`` with ``ctx_arg_name`` removed from its signature. - - Used for schema inference: the wrapper is passed to ``PythonDataFunction`` - so the context parameter is invisible to schema extraction. The original - ``fn`` is stored on the pod for actual invocation. - - Args: - fn: The original user function. - ctx_arg_name: Name of the parameter receiving ``InvocationContext``. - - Returns: - A wrapper whose ``__signature__`` and ``__annotations__`` exclude - ``ctx_arg_name``. - - Raises: - ValueError: If ``ctx_arg_name`` is not in ``fn``'s signature. - """ - sig = inspect.signature(fn) - if ctx_arg_name not in sig.parameters: - raise ValueError( - f"ctx_arg_name {ctx_arg_name!r} not found in function signature " - f"{fn.__name__!r}. Available parameters: {list(sig.parameters)}" - ) - new_params = [p for n, p in sig.parameters.items() if n != ctx_arg_name] - new_sig = sig.replace(parameters=new_params) - - @functools.wraps(fn) - def wrapper(*args, **kwargs): # pragma: no cover - return fn(*args, **kwargs) - - wrapper.__signature__ = new_sig # type: ignore[attr-defined] - wrapper.__annotations__ = { - k: v for k, v in fn.__annotations__.items() if k != ctx_arg_name - } - return wrapper -``` - -Note: `functools` is already imported at the top of the file. Add `import inspect` near the other stdlib imports. - -- [ ] **Step 2: Extend `_FunctionPodBase.__init__` to accept ctx params** - -Find the `_FunctionPodBase.__init__` method (currently ending around line 99). Modify its signature and body: - -```python -def __init__( - self, - data_function: DataFunctionProtocol, - tracker_manager: TrackerManagerProtocol | None = None, - label: str | None = None, - data_context: str | contexts.DataContext | None = None, - config: OrcapodConfig | None = None, - ctx_arg_name: str | None = None, - _original_fn: Callable | None = None, -) -> None: - super().__init__( - label=label, - data_context=data_context, - config=config, - ) - self.tracker_manager = tracker_manager or DEFAULT_TRACKER_MANAGER - self._data_function = data_function - self._post_run_hooks: list[PostRunHook] = [] - # ctx-injection support - self._ctx_arg_name: str | None = ctx_arg_name - self._original_fn: Callable | None = _original_fn - self.data_context.type_converter.ensure_types_registered_for_schemas( - data_function.input_data_schema, - data_function.output_data_schema, - ) -``` - -- [ ] **Step 3: Override `uri` property and `identity_structure()` in `_FunctionPodBase`** - -Add these overrides to `_FunctionPodBase` (after `computed_label`): - -```python -@property -def uri(self) -> tuple[str, ...]: - """Canonical URI, prefixed with ``"side_effect_function"`` when ctx is set.""" - base = self.data_function.uri - if self._ctx_arg_name is not None: - return ("side_effect_function",) + base - return base - -def identity_structure(self) -> Any: - base = self.data_function.identity_structure() - if self._ctx_arg_name is not None: - return (base, self._ctx_arg_name) - return base -``` - -Note: the existing `_FunctionPodBase` has `uri` and `identity_structure` defined — replace them (they're currently at lines ~129–131 in `function_pod.py`). - -- [ ] **Step 4: Add InvocationContext helper methods to `_FunctionPodBase`** - -Add these methods to `_FunctionPodBase` (after `pipeline_identity_structure`): - -```python -def _build_invocation_context( - self, - tag: TagProtocol, - data: DataProtocol, - *, - run_id: str | None = None, -) -> Any: - """Build a per-row ``InvocationContext`` for ctx-aware pods. - - Uses the same preimage as ``SideEffectJobNode``: system-tag columns - + ``INPUT_DATA_HASH_COL`` + ``NODE_CONTENT_HASH_COL`` + - recomputation index 0. - - Args: - tag: The input tag for this row. - data: The input data for this row. - run_id: Pipeline run identifier, or ``None`` in standalone mode. - - Returns: - An ``InvocationContext`` instance. - """ - import pyarrow as pa - from orcapod.side_effects import InvocationContext, _SIDE_EFFECT_RECOMPUTATION_INDEX_COL - - preimage = ( - tag.as_table(columns={"system_tags": True}) - .append_column( - constants.INPUT_DATA_HASH_COL, - pa.array([data.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - constants.NODE_CONTENT_HASH_COL, - pa.array([self.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - _SIDE_EFFECT_RECOMPUTATION_INDEX_COL, - pa.array([0], type=pa.int32()), - ) - ) - record_id_hash = self.data_context.arrow_hasher.hash_table(preimage) - - return InvocationContext( - pod_name=self.label, - pipeline_run_id=run_id, - _pipeline_hash_ch=self.pipeline_hash(), - _record_id_hash_ch=record_id_hash, - ) - -def _call_with_ctx(self, data: DataProtocol, ctx: Any) -> Any: - """Call the original function with data kwargs and InvocationContext. - - Args: - data: Input data row. - ctx: Per-row ``InvocationContext``. - - Returns: - Raw function return value. - - Raises: - ValueError: If ``_ctx_arg_name`` collides with a data column name. - """ - assert self._ctx_arg_name is not None and self._original_fn is not None - data_dict = data.as_dict() - if self._ctx_arg_name in data_dict: - raise ValueError( - f"ctx_arg_name {self._ctx_arg_name!r} collides with data column " - f"of the same name. Choose a different ctx_arg_name or rename the column." - ) - kwargs = {self._ctx_arg_name: ctx, **data_dict} - if inspect.iscoroutinefunction(self._original_fn): - return self._call_async_sync(kwargs) - return self._original_fn(**kwargs) - -def _call_async_sync(self, kwargs: dict[str, Any]) -> Any: - """Run the async original function synchronously. - - Args: - kwargs: Keyword arguments to pass to ``self._original_fn``. - - Returns: - The coroutine's return value. - """ - assert self._original_fn is not None - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(self._original_fn(**kwargs)) - - import concurrent.futures - with concurrent.futures.ThreadPoolExecutor(1) as executor: - future = executor.submit(lambda: asyncio.run(self._original_fn(**kwargs))) - return future.result() - -def _build_output_data(self, raw_output: Any) -> DataProtocol: - """Wrap raw function return value in a ``Data`` object with source info. - - Args: - raw_output: Raw return value from the user function. - - Returns: - A ``Data`` with source info and a new UUID. - """ - import uuid - from uuid_utils import uuid7 - from orcapod.core.datagrams import Data - from orcapod.core.data_function import parse_function_outputs - - output_keys = list(self._data_function.output_data_schema.keys()) - output_dict = parse_function_outputs(output_keys, raw_output) - new_uuid = uuid.UUID(bytes=uuid7().bytes) - source_info = { - k: f"{':'.join(self.uri)}::{new_uuid.hex}::{k}" for k in output_dict - } - return Data( - output_dict, - source_info=source_info, - record_uuid=new_uuid, - python_schema=self._data_function.output_data_schema, - data_context=self.data_context, - ) -``` - -- [ ] **Step 5: Modify `process_data` and `async_process_data` to handle ctx path** - -Replace the current `process_data` method in `_FunctionPodBase`: - -```python -def process_data( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - """Process a single data using the pod's data function. - - When ``_ctx_arg_name`` is set, builds a per-row ``InvocationContext`` - and injects it as an extra kwarg to the original function. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional ``DataExecutionLoggerProtocol`` for I/O capture. - run_id: Pipeline run identifier forwarded to ``InvocationContext``. - Only used when ``_ctx_arg_name`` is set. - - Returns: - A ``(tag, output_data)`` tuple; ``output_data`` is ``None`` if - the function filters the data out. - """ - if self._ctx_arg_name is not None: - ctx = self._build_invocation_context(tag, data, run_id=run_id) - raw = self._call_with_ctx(data, ctx) - return tag, self._build_output_data(raw) - result = self.data_function.call(data, logger=logger) - return tag, result -``` - -Replace `async_process_data`: - -```python -async def async_process_data( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - """Async counterpart of ``process_data``.""" - if self._ctx_arg_name is not None: - ctx = self._build_invocation_context(tag, data, run_id=run_id) - # _call_with_ctx handles both sync and async originals - raw = self._call_with_ctx(data, ctx) - return tag, self._build_output_data(raw) - result = await self.data_function.async_call(data, logger=logger) - return tag, result -``` - -- [ ] **Step 6: Thread `run_id` through `_invoke_with_hooks` and `_async_invoke_with_hooks`** - -Replace `_invoke_with_hooks` signature (the body calls `process_data`, so add `run_id` to it): - -```python -def _invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - if not self._post_run_hooks: - return self.process_data(tag, data, logger=logger, run_id=run_id) - - started_at = datetime.now(timezone.utc) - out_tag = tag - output_data: DataProtocol | None = None - - try: - out_tag, output_data = self.process_data(tag, data, logger=logger, run_id=run_id) - except Exception as exc: - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, None, started_at, finished_at, - InvocationStatus.ERROR, exc, - ) - ) - raise - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, output_data, started_at, finished_at, - InvocationStatus.COMPUTED, None, - ) - ) - return out_tag, output_data -``` - -Replace `_async_invoke_with_hooks` signature similarly: - -```python -async def _async_invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - if not self._post_run_hooks: - return await self.async_process_data(tag, data, logger=logger, run_id=run_id) - - started_at = datetime.now(timezone.utc) - out_tag = tag - output_data: DataProtocol | None = None - try: - out_tag, output_data = await self.async_process_data( - tag, data, logger=logger, run_id=run_id - ) - except Exception as exc: - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, None, started_at, finished_at, - InvocationStatus.ERROR, exc, - ) - ) - raise - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, output_data, started_at, finished_at, - InvocationStatus.COMPUTED, None, - ) - ) - return out_tag, output_data -``` - -- [ ] **Step 7: Add `run_id` to `_FunctionPodBase.async_execute`** - -The `async_execute` method (currently around line 396) calls `_async_invoke_with_hooks`. Add `run_id=None` and thread it through: - -```python -async def async_execute( - self, - inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - pipeline_config: PipelineConfig | None = None, - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, -) -> None: - # ... existing docstring ... - from orcapod.pipeline.observer import NoOpObserver - - try: - pipeline_config = pipeline_config or PipelineConfig() - max_concurrency = resolve_concurrency(self.pod_config, pipeline_config) - obs = observer if observer is not None else NoOpObserver() - pod_label = self.label - - sem = ( - asyncio.Semaphore(max_concurrency) - if max_concurrency is not None - else None - ) - - async def process_one(tag: TagProtocol, data: DataProtocol) -> None: - obs.on_data_start(pod_label, tag, data) - pkt_logger = obs.create_data_logger(tag, data) - try: - out_tag, result_data = await self._async_invoke_with_hooks( - tag, data, logger=pkt_logger, run_id=run_id - ) - except Exception as exc: - logger.debug( - "Data processing failed, skipping: %s", exc, exc_info=True - ) - obs.on_data_crash(pod_label, tag, data, exc) - else: - obs.on_data_end(pod_label, tag, data, result_data, cached=False) - if result_data is not None: - await output.send((out_tag, result_data)) - finally: - if sem is not None: - sem.release() - - async with asyncio.TaskGroup() as tg: - async for tag, data in inputs[0]: - if sem is not None: - await sem.acquire() - tg.create_task(process_one(tag, data)) - finally: - await output.close() -``` - -- [ ] **Step 8: Update `FunctionPod.__init__` and add `from_fn` classmethod + `side_effect_function_pod` decorator** - -Update `FunctionPod.__init__` to accept and forward the new params: - -```python -class FunctionPod(_FunctionPodBase): - def __init__( - self, - data_function: DataFunctionProtocol, - pod_config: PodConfig | None = None, - ctx_arg_name: str | None = None, - _original_fn: Callable | None = None, - **kwargs, - ) -> None: - super().__init__( - data_function, - ctx_arg_name=ctx_arg_name, - _original_fn=_original_fn, - **kwargs, - ) - self._pod_config = pod_config or PodConfig() -``` - -Add `from_fn` classmethod after the existing constructor: - -```python -@classmethod -def from_fn( - cls, - fn: Callable, - output_keys: list[str] | str, - *, - ctx_arg_name: str | None = None, - name: str | None = None, - version: str = "v1.0", - pod_config: PodConfig | None = None, - label: str | None = None, - **kwargs, -) -> "FunctionPod": - """Construct a ``FunctionPod`` directly from a callable. - - Args: - fn: The user function. - output_keys: Output column key(s). - ctx_arg_name: If set, strip this parameter from schema inference - and inject an ``InvocationContext`` per row under this name. - name: Optional canonical function name override. - version: Version string (default ``"v1.0"``). - pod_config: Optional per-pod config. - label: Optional display label. - **kwargs: Forwarded to ``_FunctionPodBase.__init__``. - - Returns: - A new ``FunctionPod``. - - Raises: - ValueError: If ``ctx_arg_name`` is set but not in ``fn``'s signature. - """ - if ctx_arg_name is not None: - original_fn = fn - stripped = _strip_ctx_from_fn(fn, ctx_arg_name) - data_function = PythonDataFunction( - stripped, - output_keys=output_keys, - function_name=name or getattr(fn, "__name__", "unknown"), - version=version, - label=label, - ) - return cls( - data_function=data_function, - pod_config=pod_config, - ctx_arg_name=ctx_arg_name, - _original_fn=original_fn, - label=label, - **kwargs, - ) - data_function = PythonDataFunction( - fn, - output_keys=output_keys, - function_name=name or getattr(fn, "__name__", "unknown"), - version=version, - label=label, - ) - return cls( - data_function=data_function, - pod_config=pod_config, - label=label, - **kwargs, - ) -``` - -Add the `side_effect_function_pod` decorator near the bottom of `function_pod.py` (after the existing `function_pod` decorator): - -```python -def side_effect_function_pod( - fn: Callable | None = None, - *, - output_keys: list[str] | str, - ctx_arg_name: str = "ctx", - name: str | None = None, - version: int = 1, - pod_config: PodConfig | None = None, -) -> "FunctionPod | Callable": - """Decorator wrapping a callable as a ctx-aware ``FunctionPod``. - - Equivalent to ``FunctionPod.from_fn(fn, output_keys=..., ctx_arg_name=...)``. - The decorated object is the ``FunctionPod`` itself (not a wrapper function), - so it can be called directly as a pod. - - Args: - fn: Optional function — if provided, decorates immediately. - output_keys: Output column key(s). - ctx_arg_name: Name of the ``InvocationContext`` parameter (default ``"ctx"``). - name: Optional canonical function name override. - version: Version integer for the URI (default 1). - pod_config: Optional per-pod configuration. - - Returns: - A ``FunctionPod`` with ``ctx_arg_name`` set, or a decorator if ``fn`` - is not provided. - - Raises: - ValueError: If ``ctx_arg_name`` is not in ``fn``'s signature. - """ - def decorator(func: Callable) -> FunctionPod: - return FunctionPod.from_fn( - func, - output_keys=output_keys, - ctx_arg_name=ctx_arg_name, - name=name, - version=f"v{version}.0", - pod_config=pod_config, - ) - - if fn is not None: - return decorator(fn) - return decorator -``` - -- [ ] **Step 9: Update `FunctionPod.process()` to use `record_function_pod_invocation` when ctx is set** - -The current `FunctionPod.process()` already calls `self.tracker_manager.record_function_pod_invocation(self, input_stream, label=label)`. This is correct — ctx-aware pods record the same invocation type. No change needed here. ✓ - -- [ ] **Step 10: Run tests to verify the new ctx code paths work** - -```bash -uv run pytest tests/test_core/function_pod/ -v -``` - -Expected: all existing tests still pass (no regressions). No new tests yet (coming in Task 13). - ---- - -### Task 2: Thread `run_id` through `CachedFunctionPod` - -**Files:** -- Modify: `src/orcapod/core/cached_function_pod.py` - -- [ ] **Step 1: Add `run_id=None` to `CachedFunctionPod.process_data`** - -```python -def process_data( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - # ... existing docstring ... - cached = self._cache.lookup(data) - if cached is not None: - module_logger.info("Pod-level cache hit") - cached = cached.with_meta_columns(**{self.RESULT_COMPUTED_FLAG: False}) - return tag, cached - - tag, output = self._function_pod.process_data(tag, data, logger=logger, run_id=run_id) - if output is not None: - pf = self._function_pod.data_function - var_dg = Datagram( - pf.get_function_variation_data(), - python_schema=pf.get_function_variation_data_schema(), - data_context=pf.data_context, - ) - exec_dg = Datagram( - pf.get_execution_data(), - python_schema=pf.get_execution_data_schema(), - data_context=pf.data_context, - ) - self._cache.store(data, output, var_dg, exec_dg) - output = output.with_meta_columns(**{self.RESULT_COMPUTED_FLAG: True}) - return tag, output -``` - -- [ ] **Step 2: Add `run_id=None` to `CachedFunctionPod.async_process_data`** - -```python -async def async_process_data( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: - # ... existing docstring ... - cached = self._cache.lookup(data) - if cached is not None: - module_logger.info("Pod-level cache hit") - cached = cached.with_meta_columns(**{self.RESULT_COMPUTED_FLAG: False}) - return tag, cached - - tag, output = await self._function_pod.async_process_data( - tag, data, logger=logger, run_id=run_id - ) - if output is not None: - pf = self._function_pod.data_function - var_dg = Datagram( - pf.get_function_variation_data(), - python_schema=pf.get_function_variation_data_schema(), - data_context=pf.data_context, - ) - exec_dg = Datagram( - pf.get_execution_data(), - python_schema=pf.get_execution_data_schema(), - data_context=pf.data_context, - ) - self._cache.store(data, output, var_dg, exec_dg) - output = output.with_meta_columns(**{self.RESULT_COMPUTED_FLAG: True}) - return tag, output -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/test_core/function_pod/ tests/test_core/data_function/ -v -``` - -Expected: all pass. - ---- - -### Task 3: Make `FunctionNodeBase.node_type` Dynamic + Add `run_id` to `FunctionJobNode` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` - -- [ ] **Step 1: Change `node_type` from class variable to instance variable** - -Currently `FunctionNodeBase` has: -```python -class FunctionNodeBase(StreamBase): - node_type = "function" -``` - -Replace with an instance attribute set in `__init__`. Change `FunctionNodeBase.__init__` body to include: - -```python -# Derive node_type from the pod's ctx_arg_name (supports is_side_effect_function_node check) -self._node_type: str = ( - "side_effect_function" - if getattr(function_pod, "_ctx_arg_name", None) is not None - else "function" -) -``` - -And add a `node_type` property: - -```python -@property -def node_type(self) -> str: - """Return ``"side_effect_function"`` when the pod injects ``InvocationContext``, else ``"function"``.""" - return self._node_type -``` - -Remove the class-level `node_type = "function"` line. - -- [ ] **Step 2: Update `FunctionJobNode.from_descriptor()` to preserve `_node_type`** - -In `from_descriptor()`, after setting other stored fields, add: -```python -# node_type comes from the stored URI: uri[0] == "side_effect_function" when ctx-aware -node._node_type = ( - "side_effect_function" - if descriptor.get("node_uri", [""])[0] == "side_effect_function" - else "function" -) -``` - -Search for the section in `from_descriptor()` that populates `_stored_*` fields and add this after them. - -- [ ] **Step 3: Add `run_id=None` to `FunctionJobNode.execute()`** - -Find `FunctionJobNode.execute()` (around line 1110). Change its signature: - -```python -def execute( - self, - input_stream: StreamProtocol, - *, - observer: ExecutionObserverProtocol | None = None, - error_policy: Literal["continue", "fail_fast"] = "continue", - run_id: str | None = None, -) -> list[tuple[TagProtocol, DataProtocol]]: -``` - -And change the `_process_data_internal` call inside it to: -```python -tag_out, result = self._process_data_internal( - tag, data, logger=pkt_logger, run_id=run_id -) -``` - -- [ ] **Step 4: Thread `run_id` through `_process_data_internal`** - -Change `_process_data_internal` signature: - -```python -def _process_data_internal( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, -) -> tuple[TagProtocol, DataProtocol | None]: -``` - -Inside, thread `run_id` to every `process_data` call: - -```python -if ephemeral_result: - ... - tag_out, output_data = self._ephemeral_cached_pod.process_data( - tag, data, logger=logger, run_id=run_id - ) - ... -elif self._cached_function_pod is not None: - tag_out, output_data = self._cached_function_pod.process_data( - tag, data, logger=logger, run_id=run_id - ) - ... -else: - tag_out, output_data = self._function_pod.process_data( - tag, data, logger=logger, run_id=run_id - ) -``` - -- [ ] **Step 5: Add `run_id=None` to `FunctionJobNode.async_execute()`** - -Find `async_execute` (around line 2047). Change signature: - -```python -async def async_execute( - self, - input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, -) -> None: -``` - -In the DB path, pass `run_id` to `execution_pod.async_execute(...)`: -```python -tg.create_task( - execution_pod.async_execute( - [compute_channel.reader], - result_channel.writer, - observer=_NodeLabelObserver(), - run_id=run_id, - ) -) -``` - -In the no-DB path, pass `run_id` to `self._function_pod.async_execute(...)`: -```python -await self._function_pod.async_execute( - [input_channel], output, observer=ctx_obs, run_id=run_id -) -``` - -- [ ] **Step 6: Run tests** - -```bash -uv run pytest tests/test_core/function_pod/ -v -``` - -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/function_pod.py src/orcapod/core/cached_function_pod.py src/orcapod/core/nodes/function_node.py -git commit -m "feat(function-pod): add ctx_arg_name support, run_id threading, and side_effect_function_pod decorator" -``` - ---- - -### Task 4: Update `FunctionNodeProtocol` in `node_protocols.py` - -**Files:** -- Modify: `src/orcapod/protocols/node_protocols.py` - -- [ ] **Step 1: Remove `SideEffectFunctionNodeProtocol` and add `run_id` to `FunctionNodeProtocol`** - -In `node_protocols.py`: - -1. Add `run_id: str | None = None` to `FunctionNodeProtocol.execute()`: -```python -def execute( - self, - input_stream: StreamProtocol, - *, - observer: ExecutionObserverProtocol | None = None, - error_policy: Literal["continue", "fail_fast"] = "continue", - run_id: str | None = None, -) -> list[tuple[TagProtocol, DataProtocol]]: ... -``` - -2. Add `run_id: str | None = None` to `FunctionNodeProtocol.async_execute()`: -```python -async def async_execute( - self, - input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, -) -> None: ... -``` - -3. Delete the entire `SideEffectFunctionNodeProtocol` class (lines 174–205). - -4. Keep `is_side_effect_function_node()` TypeGuard — it works because `FunctionJobNode` with ctx set returns `node_type == "side_effect_function"`. - -- [ ] **Step 2: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all pass (or only existing failures in SEF tests which we'll fix in Task 13). - ---- - -### Task 5: Collapse Orchestrator Dispatch - -**Files:** -- Modify: `src/orcapod/pipeline/sync_orchestrator.py` -- Modify: `src/orcapod/pipeline/async_orchestrator.py` - -- [ ] **Step 1: Merge SEF branch into function_node branch in sync orchestrator** - -In `sync_orchestrator.py`, find the dispatch loop. The current code has: -```python -elif is_function_node(node): - ... - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - error_policy=self._error_policy, - ) -... -elif is_side_effect_function_node(node): - ... - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) -``` - -After the change, both node types are `FunctionJobNode` with the same `execute()` signature. Merge by adding `run_id` to the function_node branch and removing the SEF branch: - -```python -elif is_function_node(node) or is_side_effect_function_node(node): - upstream_buf = self._gather_upstream(node, graph, buffers) - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream(upstream_buf, upstream_node) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - error_policy=self._error_policy, - run_id=run_id, - ) -``` - -Remove the `is_side_effect_function_node` import from the top of the file. - -- [ ] **Step 2: Merge SEF branch into function_node branch in async orchestrator** - -In `async_orchestrator.py`, the current code has: -```python -elif is_function_node(node): - predecessors = in_edges.get(node, []) - ... - input_reader = edge_readers[(predecessors[0], node)] - tg.create_task( - node.async_execute( - input_reader, writer, observer=effective_observer - ) - ) -... -elif is_side_effect_function_node(node): - predecessors = in_edges.get(node, []) - ... - input_reader = edge_readers[(predecessors[0], node)] - tg.create_task( - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ) - ) -``` - -Note: The SEF branch passes `[input_reader]` (a list) but `FunctionJobNode.async_execute` takes a single `input_channel`. Merge into the function_node branch, passing `run_id` and a single reader: - -```python -elif is_function_node(node) or is_side_effect_function_node(node): - predecessors = in_edges.get(node, []) - if len(predecessors) != 1: - raise ValueError( - f"FunctionNode expects exactly 1 upstream, got {len(predecessors)}" - ) - input_reader = edge_readers[(predecessors[0], node)] - tg.create_task( - node.async_execute( - input_reader, writer, observer=effective_observer, run_id=run_id - ) - ) -``` - -Remove the `is_side_effect_function_node` import from the top of the file. - -- [ ] **Step 3: Run orchestrator tests** - -```bash -uv run pytest tests/ -x -q -k "orchestrator or pipeline" -``` - -Expected: pass (or only failures in SEF-specific tests). - ---- - -### Task 6: Update `pipeline/base.py` - -Remove all SEF-specific recording, compile, and abstract property code. - -**Files:** -- Modify: `src/orcapod/pipeline/base.py` - -- [ ] **Step 1: Remove SEF imports and abstract property** - -1. Remove `SideEffectFunctionInvocation` from the imports at the top. -2. Remove the `side_effect_function_node_class` abstract property: -```python -# DELETE this entire block: -@property -@abstractmethod -def side_effect_function_node_class(self) -> type: - """Node class for side-effect-function pod invocations.""" - ... -``` - -- [ ] **Step 2: Remove `record_side_effect_function_pod_invocation`** - -Delete the entire `record_side_effect_function_pod_invocation()` method from `AbstractPipelineBase`. - -- [ ] **Step 3: Remove SEF branch from `compile()`** - -In `compile()`, find: -```python -elif isinstance(inv, SideEffectFunctionInvocation): - node_map[key] = self.side_effect_function_node_class( - pod=inv.pod, - input_stream=upstream_nodes[0], - label=inv.label, - ) -``` - -Delete this `elif` block entirely. `FunctionInvocation` now covers ctx-aware pods too (since `FunctionPod.process()` calls `record_function_pod_invocation` regardless). - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - ---- - -### Task 7: Update `pipeline/pod_invocation.py` - -**Files:** -- Modify: `src/orcapod/pipeline/pod_invocation.py` - -- [ ] **Step 1: Remove `SideEffectFunctionInvocation` class** - -Delete the entire `SideEffectFunctionInvocation` class (the last class in the file) and its `SideEffectFunctionPodProtocol` import. - -```python -# DELETE from TYPE_CHECKING imports: -from orcapod.protocols.core_protocols import ( - ..., - SideEffectFunctionPodProtocol, # <- remove this line - ... -) - -# DELETE the entire class: -class SideEffectFunctionInvocation(PodInvocation): - ... -``` - -- [ ] **Step 2: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - ---- - -### Task 8: Update `pipeline/job.py` - -**Files:** -- Modify: `src/orcapod/pipeline/job.py` - -- [ ] **Step 1: Remove SEF imports and class attribute** - -1. Remove the import: -```python -# DELETE: -from orcapod.core.side_effect_function.side_effect_function_pod import ( - SideEffectFunctionJobNode, - SideEffectFunctionNode, -) -``` - -2. Remove the class attribute: -```python -# DELETE: -side_effect_function_node_class = SideEffectFunctionJobNode -``` - -- [ ] **Step 2: Remove SEF branch from `_distribute_databases()`** - -Find: -```python -elif isinstance(node, SideEffectFunctionJobNode): - node.attach_databases(result_database=result_db) -``` - -Delete this `elif` block. `FunctionJobNode` with ctx-aware pods is now wired the same as regular `FunctionJobNode`. - -- [ ] **Step 3: Remove SEF branch from `as_pipeline()`** - -Find: -```python -elif isinstance(job_node, SideEffectFunctionJobNode): - upstream_bp_hash = job_id_to_bp_hash[id(job_node._input_stream)] - node_map[node_hash] = SideEffectFunctionNode( - pod=job_node._pod, - input_stream=node_map[upstream_bp_hash], - label=job_node._label, - ) -``` - -Delete this `elif` block. Ctx-aware `FunctionJobNode` are converted to `FunctionNode` by the existing `FunctionJobNode` branch (since `job_node._function_pod` holds the ctx-aware `FunctionPod`). - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - ---- - -### Task 9: Update `pipeline/graph.py` - -**Files:** -- Modify: `src/orcapod/pipeline/graph.py` - -- [ ] **Step 1: Remove SEF import and class attribute** - -1. Remove: -```python -from orcapod.core.side_effect_function.side_effect_function_pod import SideEffectFunctionNode -``` - -2. Remove: -```python -side_effect_function_node_class = SideEffectFunctionNode -``` - -- [ ] **Step 2: Update `save()` to remove SEF node_type check** - -Find: -```python -case _ if node.node_type == "side_effect_function": - raise NotImplementedError( - f"Pipeline.save() does not support SideEffectFunctionNode ..." - ) -``` - -Delete this `case` block. Ctx-aware `FunctionNode` objects are now serialized via the `FunctionNode()` case (which already exists and handles `node._function_pod.to_config()`). - -Note: `FunctionNode.to_config()` currently exists. For ctx-aware `FunctionPod`, `to_config()` should include `ctx_arg_name`. We'll need to update `FunctionPod.to_config()` and `FunctionPod.from_config()` to handle `ctx_arg_name`. Add to `FunctionPod.to_config()`: -```python -def to_config(self) -> dict[str, Any]: - config: dict[str, Any] = { - "uri": list(self.uri), - "data_function": self.data_function.to_config(), - "pod_config": None, - "ctx_arg_name": self._ctx_arg_name, # NEW - } - if self._pod_config.max_concurrency is not None: - config["pod_config"] = {"max_concurrency": self._pod_config.max_concurrency} - return config -``` - -For `from_config()`: currently ctx-aware pods require a live callable; `from_config()` is only called for deserialization (read-only stubs), so `ctx_arg_name` can be stored but `_original_fn` stays None. The deserialized node won't be able to compute, only serve cached results — which is consistent with `LoadStatus.READ_ONLY` behavior. Update `from_config()`: -```python -@classmethod -def from_config(cls, config: dict[str, Any], *, fallback_to_proxy: bool = False) -> "FunctionPod": - from orcapod.pipeline.serialization import resolve_data_function_from_config - - pf_config = config["data_function"] - data_function = resolve_data_function_from_config(pf_config, fallback_to_proxy=fallback_to_proxy) - - pod_config = None - if config.get("pod_config") is not None: - pod_config = PodConfig(**config["pod_config"]) - - return cls( - data_function=data_function, - pod_config=pod_config, - ctx_arg_name=config.get("ctx_arg_name"), - # _original_fn stays None — this is a read-only stub - ) -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - ---- - -### Task 10: Update `tracker.py` and Protocol Files - -**Files:** -- Modify: `src/orcapod/core/tracker.py` -- Modify: `src/orcapod/protocols/core_protocols/trackers.py` -- Modify: `src/orcapod/protocols/core_protocols/__init__.py` -- Delete: `src/orcapod/protocols/core_protocols/side_effect_function_pod.py` - -- [ ] **Step 1: Remove `record_side_effect_function_pod_invocation` from `BasicTrackerManager`** - -In `tracker.py`, delete: -```python -def record_side_effect_function_pod_invocation( - self, - pod: Any, - input_stream: cp.StreamProtocol, - label: str | None = None, -) -> None: - """Record a side-effect function pod invocation in all active trackers.""" - for tracker in self.get_active_trackers(): - tracker.record_side_effect_function_pod_invocation(pod, input_stream, label=label) -``` - -- [ ] **Step 2: Remove from `AutoRegisteringContextBasedTracker`** - -Delete the abstract method: -```python -@abstractmethod -def record_side_effect_function_pod_invocation( - self, - pod: Any, - input_stream: cp.StreamProtocol, - label: str | None = None, -) -> None: ... -``` - -- [ ] **Step 3: Remove from `trackers.py` protocol** - -In `src/orcapod/protocols/core_protocols/trackers.py`, remove `record_side_effect_function_pod_invocation` from both protocol classes (around line 112 and 241). - -- [ ] **Step 4: Delete `side_effect_function_pod.py` and update `__init__.py`** - -```bash -rm src/orcapod/protocols/core_protocols/side_effect_function_pod.py -``` - -In `src/orcapod/protocols/core_protocols/__init__.py`, remove: -```python -from .side_effect_function_pod import SideEffectFunctionPodProtocol -``` - -And remove `SideEffectFunctionPodProtocol` from `__all__` if present. - -- [ ] **Step 5: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - ---- - -### Task 11: Update `orcapod/__init__.py` - -**Files:** -- Modify: `src/orcapod/__init__.py` - -- [ ] **Step 1: Remove SEF class exports, keep `side_effect_function_pod` decorator** - -Remove: -```python -from .core.side_effect_function import ( - SideEffectFunctionPod, - SideEffectFunctionNode, - SideEffectFunctionJobNode, - side_effect_function_pod, -) -``` - -Add import of the new decorator from `function_pod.py`: -```python -from .core.function_pod import ( - FunctionPod, - function_pod, - side_effect_function_pod, # thin alias for FunctionPod.from_fn with ctx_arg_name -) -``` - -Remove from `__all__`: -- `"SideEffectFunctionPod"` -- `"SideEffectFunctionNode"` -- `"SideEffectFunctionJobNode"` - -Keep `"side_effect_function_pod"` in `__all__`. - -- [ ] **Step 2: Run tests** - -```bash -uv run pytest tests/ -x -q -``` - ---- - -### Task 12: Delete `src/orcapod/core/side_effect_function/` Module - -**Files:** -- Delete: `src/orcapod/core/side_effect_function/__init__.py` -- Delete: `src/orcapod/core/side_effect_function/side_effect_function_pod.py` -- Delete the directory: `src/orcapod/core/side_effect_function/` - -- [ ] **Step 1: Delete the SEF module** - -```bash -rm -rf src/orcapod/core/side_effect_function/ -``` - -- [ ] **Step 2: Verify no remaining imports** - -```bash -grep -r "side_effect_function" src/orcapod/ --include="*.py" | grep -v "__pycache__" -``` - -Expected: only occurrences in `function_pod.py` (the new `side_effect_function_pod` decorator) and maybe the `node_type` value `"side_effect_function"` in `function_node.py` and `node_protocols.py`. - -- [ ] **Step 3: Run all tests** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests pass except the SEF-specific test file (which we'll update in Task 13). - -- [ ] **Step 4: Commit** - -```bash -git add -A -git commit -m "refactor(function-pod): eliminate SideEffectFunctionPod hierarchy, unify into FunctionPod" -``` - ---- - -### Task 13: Update Tests to Use Unified API - -**Files:** -- Modify: `tests/test_core/side_effect_function/test_side_effect_function_pod.py` - -All references to `SideEffectFunctionPod`, `SideEffectFunctionJobNode`, etc. must be updated. The complete updated test file: - -- [ ] **Step 1: Write the updated test file** - -Replace the entire file with: - -```python -# tests/test_core/side_effect_function/test_side_effect_function_pod.py -from __future__ import annotations - -import pytest -import pyarrow as pa - -from orcapod.core.streams import ArrowTableStream -from orcapod.side_effects import InvocationContext - - -def _make_stream(n: int = 3) -> ArrowTableStream: - """Simple stream: tag=id (int), data=value (int).""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("value", pa.int64(), nullable=False), - ]) - table = pa.table( - {"id": list(range(n)), "value": list(range(n))}, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - -def _make_pipeline_db(): - """Return a fresh in-memory ArrowDatabase.""" - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase - return InMemoryArrowDatabase() - - -class TestSideEffectFunctionPodSchema: - """SF-01, SF-02, SF-03, SF-10: schema inference and ctx stripping.""" - - def test_sf01_ctx_stripped_from_input_schema(self): - """SF-01: 'ctx' param stripped; data params form the input schema.""" - from orcapod.core.function_pod import FunctionPod - - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"result_{value}" - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - - # Input schema excludes 'ctx' - assert "ctx" not in pod.input_data_schema - assert "value" in pod.input_data_schema - assert pod.input_data_schema["value"] == int - - # Output schema has the declared key - assert "result" in pod.output_data_schema - assert pod.output_data_schema["result"] == str - - def test_sf02_custom_ctx_arg_name(self): - """SF-02: ctx_arg_name='context' — stripped and injected by correct name.""" - from orcapod.core.function_pod import FunctionPod - - def my_fn(value: int, context: InvocationContext) -> str: - return f"r_{value}" - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="context") - assert "context" not in pod.input_data_schema - assert "value" in pod.input_data_schema - - def test_sf03_missing_ctx_arg_raises_at_construction(self): - """SF-03: Missing ctx_arg_name raises ValueError at construction time.""" - from orcapod.core.function_pod import FunctionPod - - def my_fn(value: int) -> str: - return str(value) - - with pytest.raises(ValueError, match="ctx_arg_name"): - FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - - def test_sf10_node_uri_shape(self): - """SF-10: uri[0]=='side_effect_function', uri[-1]=='python.function.v0', len==5.""" - from orcapod.core.function_pod import FunctionPod - - def my_fn(value: int, ctx: InvocationContext) -> str: - return str(value) - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - assert pod.uri[0] == "side_effect_function" - assert pod.uri[-1] == "python.function.v0" - assert len(pod.uri) == 5 - assert pod.uri[3] == "v1.0" - - -class TestSideEffectFunctionPodStreamStandalone: - """SF-04, SF-05: standalone execution via FunctionPodStream.""" - - def test_sf04_iter_data_returns_correct_output(self): - """SF-04: iter_data() returns correct (tag, output_data) per row.""" - from orcapod.core.function_pod import FunctionPod - - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"v{value}" - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(3) - rows = list(pod.process(stream).iter_data()) - - assert len(rows) == 3 - for i, (tag, data) in enumerate(rows): - assert data.as_dict()["result"] == f"v{i}" - # Tags pass through unchanged - assert rows[0][0].as_dict()["id"] == 0 - assert rows[1][0].as_dict()["id"] == 1 - - def test_sf05_invocation_context_fields_standalone(self): - """SF-05: InvocationContext has pod_name, non-empty hash, pipeline_run_id=None.""" - from orcapod.core.function_pod import FunctionPod - - received_ctx: list[InvocationContext] = [] - - def my_fn(value: int, ctx: InvocationContext) -> str: - received_ctx.append(ctx) - return str(value) - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(1) - list(pod.process(stream).iter_data()) - - assert len(received_ctx) == 1 - ctx = received_ctx[0] - assert ctx.pod_name == pod.label - assert isinstance(ctx.invocation_hash, str) - assert len(ctx.invocation_hash) > 0 - assert "::" in ctx.invocation_hash - assert ctx.pipeline_run_id is None # standalone: no run_id - - def test_sf05b_async_fn_routed_through_sync_execute(self): - """SF-05b: async user function executed correctly via _call_async_sync.""" - from orcapod.core.function_pod import FunctionPod - - import asyncio - - async def my_async_fn(value: int, ctx: InvocationContext) -> str: - await asyncio.sleep(0) - return f"async_{value}" - - pod = FunctionPod.from_fn(my_async_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(2) - rows = list(pod.process(stream).iter_data()) - - assert len(rows) == 2 - assert rows[0][1].as_dict()["result"] == "async_0" - assert rows[1][1].as_dict()["result"] == "async_1" - - -class TestSideEffectFunctionJobNode: - """SF-06, SF-07, SF-09: DB-backed sync execution via FunctionJobNode.""" - - def test_sf06_output_cached_after_first_run(self): - """SF-06: Output cached; second run returns cached result without re-calling fn.""" - from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes.function_node import FunctionJobNode - - call_count = 0 - - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"r{value}" - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(2) - pipeline_db = _make_pipeline_db() - - node1 = FunctionJobNode(function_pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=pipeline_db) - results1 = node1.execute(stream) - assert len(results1) == 2 - assert call_count == 2 - - # Second run — same pod, same data, same DB — fn must NOT be called again - node2 = FunctionJobNode(function_pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=pipeline_db) - results2 = node2.execute(stream) - assert len(results2) == 2 - assert call_count == 2 # NOT incremented — cache hit - - # Both runs produce equal result values - for (_, d1), (_, d2) in zip(results1, results2): - assert d1.as_dict()["result"] == d2.as_dict()["result"] - - def test_sf07_data_function_accessible_and_uri_consistent(self): - """SF-07: pod._data_function is a PythonDataFunction; uri starts with 'side_effect_function'.""" - from orcapod.core.function_pod import FunctionPod - from orcapod.core.data_function import PythonDataFunction - - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"r{value}" - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - assert isinstance(pod._data_function, PythonDataFunction) - assert pod.uri[0] == "side_effect_function" - assert pod._ctx_arg_name == "ctx" - - def test_sf09_on_error_reraises(self): - """SF-09: exceptions from user function always propagate.""" - from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes.function_node import FunctionJobNode - - def my_fn(value: int, ctx: InvocationContext) -> str: - raise RuntimeError("test error") - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(1) - pipeline_db = _make_pipeline_db() - node = FunctionJobNode(function_pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=pipeline_db) - - # Must propagate — no silent row suppression - with pytest.raises(RuntimeError, match="test error"): - node.execute(stream) - - -class TestSideEffectFunctionPodDecorator: - """SF-11: @side_effect_function_pod decorator.""" - - def test_sf11_decorator_creates_correct_pod(self): - """SF-11: Decorator creates a FunctionPod with correct URI.""" - from orcapod.core.function_pod import FunctionPod, side_effect_function_pod - - @side_effect_function_pod(output_keys=["result"]) - def my_fn(value: int, ctx: InvocationContext) -> str: - return str(value) - - assert isinstance(my_fn, FunctionPod) - assert my_fn.uri[0] == "side_effect_function" - assert my_fn.canonical_function_name == "my_fn" - - def test_sf11_decorator_accessible_from_public_api(self): - """SF-11: Decorator accessible from orcapod top-level; class removed.""" - import orcapod - assert hasattr(orcapod, "side_effect_function_pod") - assert hasattr(orcapod, "FunctionPod") - # SideEffectFunctionPod class is no longer exported (unified into FunctionPod) - assert not hasattr(orcapod, "SideEffectFunctionPod") - - -class TestSideEffectFunctionPodPipelineIntegration: - """SF-12: Full pipeline compilation and execution.""" - - def test_sf12_pipeline_compilation_and_execution(self): - """SF-12: FunctionJobNode compiled for ctx-aware pod, fn called, ctx received.""" - from orcapod.pipeline.job import PipelineJob - from orcapod.core.function_pod import FunctionPod - from orcapod.core.sources.dict_source import DictSource - - received_ctx: list[InvocationContext] = [] - - def transform(value: int, ctx: InvocationContext) -> str: - received_ctx.append(ctx) - return f"result_{value}" - - pod = FunctionPod.from_fn(transform, output_keys=["result"], ctx_arg_name="ctx") - db = _make_pipeline_db() - - with PipelineJob(name="test_sef", store=db) as job: - source = DictSource( - [{"id": 0, "value": 10}, {"id": 1, "value": 20}], - tag_columns=["id"], - ) - pod.process(source) - - job.run() - - assert len(received_ctx) == 2 # fn called once per row - - # Find the side_effect_function node in the compiled graph - sef_nodes = [ - n for n in job.dag.nodes() - if getattr(n, "node_type", None) == "side_effect_function" - ] - assert len(sef_nodes) == 1 - - def test_sf12_second_pipeline_run_uses_cache(self): - """SF-12: Second pipeline run uses cached output; fn not called again.""" - from orcapod.pipeline.job import PipelineJob - from orcapod.core.function_pod import FunctionPod - from orcapod.core.sources.dict_source import DictSource - - call_count = 0 - - def transform(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"r{value}" - - pod = FunctionPod.from_fn(transform, output_keys=["result"], ctx_arg_name="ctx") - db = _make_pipeline_db() - source_data = [{"id": 0, "value": 10}, {"id": 1, "value": 20}] - - with PipelineJob(name="test_sef_cache", store=db) as job1: - source1 = DictSource(source_data, tag_columns=["id"]) - pod.process(source1) - job1.run() - assert call_count == 2 - - with PipelineJob(name="test_sef_cache", store=db) as job2: - source2 = DictSource(source_data, tag_columns=["id"]) - pod.process(source2) - job2.run() - assert call_count == 2 # NOT called again — cache hit - - -class TestSideEffectFunctionJobNodeAsync: - """SF-13: async_execute produces same output as sync path.""" - - def test_sf13_async_execute_basic(self): - """SF-13: async_execute processes all rows, writes cache, returns correct output.""" - import asyncio - from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes.function_node import FunctionJobNode - from orcapod.channels import Channel - - call_count = 0 - - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"async_{value}" - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(3) - pipeline_db = _make_pipeline_db() - node = FunctionJobNode(function_pod=pod, input_stream=stream) - node.attach_databases(pipeline_database=pipeline_db) - - async def _run(): - ch_in = Channel(buffer_size=10) - ch_out = Channel(buffer_size=10) - - async def feed(): - for tag, data in stream.iter_data(): - await ch_in.writer.send((tag, data)) - await ch_in.writer.close() - - await asyncio.gather( - feed(), - node.async_execute( - ch_in.reader, ch_out.writer, run_id="test-run-sf13" - ), - ) - return await ch_out.reader.collect() - - results = asyncio.run(_run()) - assert len(results) == 3 - assert call_count == 3 - - # Verify output values - output_values = [data.as_dict()["result"] for _, data in results] - for i in range(3): - assert f"async_{i}" in output_values -``` - -- [ ] **Step 2: Run the updated test file** - -```bash -uv run pytest tests/test_core/side_effect_function/test_side_effect_function_pod.py -v -``` - -Expected: all 13 test cases pass. - -- [ ] **Step 3: Run the full test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: all tests pass. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_core/side_effect_function/test_side_effect_function_pod.py -git commit -m "test(function-pod): update SEF tests to use unified FunctionPod.from_fn API" -``` - ---- - -### Task 14: Close PR #230 and Create New PR - -- [ ] **Step 1: Close the old PR** - -```bash -gh pr close 230 --comment "Superseded by unified FunctionPod design (ITL-532). This PR's approach has been replaced by ctx_arg_name support on FunctionPod directly." -``` - -- [ ] **Step 2: Push the new branch** - -```bash -git push -u origin HEAD -``` - -- [ ] **Step 3: Create the new PR** - -```bash -gh pr create --title "feat(function-pod): unified ctx-injection via FunctionPod.from_fn (ITL-532)" --body "$(cat <<'EOF' -## Summary - -- Eliminates the separate `SideEffectFunctionPod` class hierarchy -- `InvocationContext` injection is now an optional feature of `FunctionPod` via `ctx_arg_name` -- When `ctx_arg_name` is set, `uri` is prefixed with `"side_effect_function"`, `node_type` returns `"side_effect_function"`, and `process_data()` builds and injects `InvocationContext` per row -- `FunctionJobNode.execute()` and `async_execute()` gain `run_id: str | None = None` threaded to ctx building -- `@side_effect_function_pod` decorator kept as a thin alias of `FunctionPod.from_fn` -- Entire `src/orcapod/core/side_effect_function/` module deleted -- `SideEffectFunctionNodeProtocol`, `SideEffectFunctionInvocation`, `record_side_effect_function_pod_invocation`, and `side_effect_function_node_class` all removed - -Closes ITL-532 - -## Test plan -- [ ] `uv run pytest tests/test_core/side_effect_function/ -v` — all 13 SF tests pass -- [ ] `uv run pytest tests/ -v` — full suite passes - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - ---- - -## Self-Review Checklist - -**Spec coverage:** -- ✅ `ctx_arg_name` support in `FunctionPod` — Task 1 -- ✅ URI prefixed with `"side_effect_function"` — Task 1 Step 3 -- ✅ `identity_structure()` includes `ctx_arg_name` — Task 1 Step 3 -- ✅ `node_type` returns `"side_effect_function"` dynamically — Task 3 Step 1 -- ✅ `process_data()` builds InvocationContext and injects it — Task 1 Steps 4–5 -- ✅ `run_id` on `FunctionJobNode.execute()` and `async_execute()` — Task 3 Steps 3–5 -- ✅ `CachedFunctionPod` threads `run_id` — Task 2 -- ✅ `_strip_ctx_from_fn` moved to `function_pod.py` — Task 1 Step 1 -- ✅ `side_effect_function/` module deleted — Task 12 -- ✅ `SideEffectFunctionNodeProtocol` deleted — Task 4 -- ✅ `record_side_effect_function_pod_invocation` deleted — Task 10 -- ✅ `SideEffectFunctionInvocation` deleted — Task 7 -- ✅ `side_effect_function_node_class` deleted — Tasks 6, 8, 9 -- ✅ Orchestrator dispatch collapsed — Task 5 -- ✅ `@side_effect_function_pod` decorator kept as alias — Task 1 Step 8 -- ✅ Tests updated — Task 13 -- ✅ PR #230 closed — Task 14 - -**Type consistency check:** -- `FunctionPod.from_fn(fn, output_keys, *, ctx_arg_name, ...)` — used in all tests ✓ -- `FunctionJobNode(function_pod=pod, input_stream=stream)` — consistent ✓ -- `node.attach_databases(pipeline_database=pipeline_db)` — consistent ✓ -- `node.async_execute(ch_in.reader, ch_out.writer, run_id=...)` — single reader, consistent with `FunctionJobNode.async_execute` signature ✓ -- `process_data(tag, data, *, logger=None, run_id=None)` — consistent across `_FunctionPodBase`, `CachedFunctionPod` ✓ - -**`uri[3] == "v1.0"` note:** Test SF-10 currently checks `pod.uri[3] == "v1"`. After the change, `from_fn` uses `version="v1.0"` by default. The `PythonDataFunction` stores the version string verbatim in the URI. If the existing `SideEffectFunctionPod` used `version=f"v{version}.0"` (i.e., `"v1.0"` for the default `version=1`), then the URI would be `("side_effect_function", ..., "v1.0", "python.function.v0")`. The test expects `pod.uri[3] == "v1"` — this may need adjustment. In `from_fn`, use `version=f"v{version}.0"` when `version` is an int. In the classmethod, since `version` is already a string default `"v1.0"`, `uri[3]` would be `"v1.0"`. Update the test to check `pod.uri[3] == "v1.0"`. diff --git a/superpowers/plans/2026-07-17-ctx-arg-and-ray-opts-hardening.md b/superpowers/plans/2026-07-17-ctx-arg-and-ray-opts-hardening.md deleted file mode 100644 index 537a897a1..000000000 --- a/superpowers/plans/2026-07-17-ctx-arg-and-ray-opts-hardening.md +++ /dev/null @@ -1,805 +0,0 @@ -# ctx_arg + Ray empty-opts hardening implementation plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix two failing Ray executor tests, add full test coverage for `@function_pod(ctx_arg=...)` as a first-class path (10 scenarios including cached pod wrapping), and update the `side_effect_function_pod` docstring to point to the preferred path. - -**Architecture:** All implementation already exists; this plan is test + doc work. The Ray fix updates two test mocks to handle a dual calling convention. The ctx_arg tests are written to the new file `tests/test_core/function_pod/test_function_pod_ctx_arg.py`, mirroring the structure of `tests/test_core/side_effect_function/test_side_effect_function_pod.py`. The docstring update touches only `src/orcapod/core/function_pod.py`. - -**Tech Stack:** Python, pytest (`uv run pytest`), PyArrow, orcapod internal APIs. - ---- - -## File Map - -| Action | Path | Responsibility | -|--------|------|----------------| -| Modify | `tests/test_core/test_regression_fixes.py` | Fix two Ray mocks — dual calling convention | -| Modify | `src/orcapod/core/function_pod.py` | Update `side_effect_function_pod` docstring | -| Create | `tests/test_core/function_pod/test_function_pod_ctx_arg.py` | 10 ctx_arg test scenarios | - ---- - -## Task 1: Fix `test_get_remote_fn_caches_per_function_name` - -**Files:** -- Modify: `tests/test_core/test_regression_fixes.py` (around line 591) - -**Context:** The guard in `RayExecutor._get_remote_fn()` calls `ray.remote(wrapper)` directly -when `opts` is empty. The current mock sets `mock_ray.remote.return_value = lambda wrapper: ...`, -which means `mock_ray.remote(wrapper)` returns the lambda itself — not `lambda(wrapper)`. -Replace the `return_value` assignment with a function that handles both call patterns. - -- [ ] **Step 1: Read the test to confirm the exact lines to change** - - Open `tests/test_core/test_regression_fixes.py` lines 591–616. Locate: - ```python - mock_ray.remote.return_value = lambda wrapper: MagicMock(name=f"remote_{wrapper.__name__}") - ``` - -- [ ] **Step 2: Replace the mock with a dual-pattern fake** - - Replace those lines so the full test reads: - ```python - def test_get_remote_fn_caches_per_function_name(self): - """_get_remote_fn must return distinct remote wrappers for different - function names so Ray metrics report the correct name.""" - from unittest.mock import MagicMock, patch - - from orcapod.core.executors.ray import RayExecutor - - mock_ray = MagicMock() - mock_ray.is_initialized.return_value = True - - def fake_remote(fn=None, **opts): - # Called as ray.remote(fn) when opts is empty (direct path) - if fn is not None: - return MagicMock(name=f"remote_{fn.__name__}") - # Called as ray.remote(**opts) when opts is non-empty (factory path) - return lambda wrapper: MagicMock(name=f"remote_{wrapper.__name__}") - - mock_ray.remote = fake_remote - - with patch.dict("sys.modules", {"ray": mock_ray}): - executor = RayExecutor.__new__(RayExecutor) - executor._remote_opts = {} - executor._remote_fn_cache = {} - executor._remote_fn_cache_lock = __import__("threading").Lock() - - fn_a = executor._get_remote_fn(mock_ray, "transform_a") - fn_b = executor._get_remote_fn(mock_ray, "transform_b") - fn_a_again = executor._get_remote_fn(mock_ray, "transform_a") - - # Different names → different remote fns - assert fn_a is not fn_b - # Same name → cached (same object) - assert fn_a is fn_a_again - ``` - -- [ ] **Step 3: Run this test alone to verify it passes** - - ```bash - uv run pytest tests/test_core/test_regression_fixes.py::TestRayExecutorRegressions::test_get_remote_fn_caches_per_function_name -v - ``` - Expected: `PASSED` - ---- - -## Task 2: Fix `test_get_remote_fn_sets_wrapper_name` - -**Files:** -- Modify: `tests/test_core/test_regression_fixes.py` (around line 618) - -**Context:** Same root cause as Task 1. `fake_remote` only accepts `**opts`; when the guard -calls `ray.remote(wrapper)` directly (positional argument), Python raises `TypeError: -fake_remote() takes 0 positional arguments but 1 was given`. - -- [ ] **Step 1: Locate the `fake_remote` definition** (lines 627–631): - ```python - def fake_remote(**opts): - def decorator(wrapper): - captured_wrappers.append(wrapper) - return MagicMock() - return decorator - ``` - -- [ ] **Step 2: Replace with a dual-pattern fake** - - Replace the full test with: - ```python - def test_get_remote_fn_sets_wrapper_name(self): - """The capture wrapper created by _get_remote_fn should carry the - original function name so Ray uses it in metrics labels.""" - from unittest.mock import MagicMock, patch - - from orcapod.core.executors.ray import RayExecutor - - captured_wrappers = [] - - def fake_remote(fn=None, **opts): - # Called as ray.remote(fn) when opts is empty (direct path) - if fn is not None: - captured_wrappers.append(fn) - return MagicMock() - # Called as ray.remote(**opts) when opts is non-empty (factory path) - def decorator(wrapper): - captured_wrappers.append(wrapper) - return MagicMock() - return decorator - - mock_ray = MagicMock() - mock_ray.is_initialized.return_value = True - mock_ray.remote = fake_remote - - with patch.dict("sys.modules", {"ray": mock_ray}): - executor = RayExecutor.__new__(RayExecutor) - executor._remote_opts = {} - executor._remote_fn_cache = {} - executor._remote_fn_cache_lock = __import__("threading").Lock() - - executor._get_remote_fn(mock_ray, "compute_features") - - assert len(captured_wrappers) == 1 - assert captured_wrappers[0].__name__ == "compute_features" - ``` - -- [ ] **Step 3: Run both Ray tests together** - - ```bash - uv run pytest tests/test_core/test_regression_fixes.py::TestRayExecutorRegressions::test_get_remote_fn_caches_per_function_name tests/test_core/test_regression_fixes.py::TestRayExecutorRegressions::test_get_remote_fn_sets_wrapper_name -v - ``` - Expected: both `PASSED` - -- [ ] **Step 4: Run the full regression test suite to confirm no regressions** - - ```bash - uv run pytest tests/test_core/test_regression_fixes.py -v - ``` - Expected: all pass - -- [ ] **Step 5: Commit** - - ```bash - git add tests/test_core/test_regression_fixes.py - git commit -m "test(ray): fix two Ray executor mocks to handle empty-opts direct call path (ITL-544)" - ``` - ---- - -## Task 3: Update `side_effect_function_pod` docstring - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` (around line 1220) - -**Context:** The `side_effect_function_pod` decorator is superseded by `@function_pod(ctx_arg=...)`. -Update its docstring to note this with a before/after example. No runtime warning — this is -a greenfield pre-v0.1.0 project. - -- [ ] **Step 1: Locate the current docstring** (lines 1228–1252): - - The current docstring starts: - ``` - """Decorator wrapping a callable as a ctx-aware ``FunctionPod``. - - Equivalent to ``FunctionPod.from_fn(fn, output_keys=..., ctx_arg_name=...)``. - ``` - -- [ ] **Step 2: Replace the docstring** - - Replace the docstring so it reads (keep all Args/Returns/Raises intact, add the note at the top): - ```python - """Decorator wrapping a callable as a ctx-aware ``FunctionPod``. - - .. note:: - This decorator is superseded by ``@function_pod(ctx_arg=)``, - which is now the preferred way to author side-effect pods. The two - forms are equivalent in behaviour. - - Preferred form:: - - @function_pod(output_keys=["result"], ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - ... - - assert isinstance(my_fn.pod, FunctionPod) - - Legacy form (this decorator):: - - @side_effect_function_pod(output_keys=["result"]) - def my_fn(value: int, ctx: InvocationContext) -> str: - ... - - assert isinstance(my_fn, FunctionPod) - - Full removal of this decorator is tracked in a follow-up issue. - - Equivalent to ``FunctionPod.from_fn(fn, output_keys=..., ctx_arg_name=...)``. - The decorated object is the ``FunctionPod`` itself (not a wrapper function), - so it can be called directly as a pod. - - Args: - fn: Optional function — if provided, decorates immediately. - output_keys: Output column key(s). - ctx_arg_name: Name of the ``InvocationContext`` parameter (default ``"ctx"``). - name: Optional canonical function name override. - version: Version string for the data function (default ``"v1.0"``). - pod_config: Optional per-pod configuration. - - Returns: - A ``FunctionPod`` with ``ctx_arg_name`` set, or a decorator if ``fn`` - is not provided. - - Raises: - ValueError: If ``ctx_arg_name`` is not in ``fn``'s signature. - """ - ``` - - Note: Google-style docstrings don't use `.. note::` RST directives — use a plain - `Note:` section instead. The final docstring should be: - ```python - """Decorator wrapping a callable as a ctx-aware ``FunctionPod``. - - Note: - This decorator is superseded by ``@function_pod(ctx_arg=)``, - which is now the preferred way to author side-effect pods. The two - forms are equivalent in behaviour. - - Preferred form (use this instead):: - - @function_pod(output_keys=["result"], ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - ... - - assert isinstance(my_fn.pod, FunctionPod) - - Legacy form (this decorator):: - - @side_effect_function_pod(output_keys=["result"]) - def my_fn(value: int, ctx: InvocationContext) -> str: - ... - - assert isinstance(my_fn, FunctionPod) - - Full removal of this decorator is tracked separately. - - Equivalent to ``FunctionPod.from_fn(fn, output_keys=..., ctx_arg_name=...)``. - The decorated object is the ``FunctionPod`` itself (not a wrapper function), - so it can be called directly as a pod. - - Args: - fn: Optional function — if provided, decorates immediately. - output_keys: Output column key(s). - ctx_arg_name: Name of the ``InvocationContext`` parameter (default ``"ctx"``). - name: Optional canonical function name override. - version: Version string for the data function (default ``"v1.0"``). - pod_config: Optional per-pod configuration. - - Returns: - A ``FunctionPod`` with ``ctx_arg_name`` set, or a decorator if ``fn`` - is not provided. - - Raises: - ValueError: If ``ctx_arg_name`` is not in ``fn``'s signature. - """ - ``` - -- [ ] **Step 3: Run the existing side_effect tests to confirm nothing broke** - - ```bash - uv run pytest tests/test_core/side_effect_function/ -v - ``` - Expected: all pass (no code changed, only docstring) - -- [ ] **Step 4: Commit** - - ```bash - git add src/orcapod/core/function_pod.py - git commit -m "docs(function-pod): mark side_effect_function_pod as superseded by ctx_arg path (ITL-544)" - ``` - ---- - -## Task 4: Create ctx_arg test file — schema scenarios (1–2) - -**Files:** -- Create: `tests/test_core/function_pod/test_function_pod_ctx_arg.py` - -**Context:** The `@function_pod(ctx_arg=...)` decorator (already implemented) wraps the -function in a `FunctionPod` with `ctx_arg_name` set. The pod filters `ctx` from its exposed -`input_data_schema` while the underlying `_data_function.input_data_schema` retains it for -hashing. These tests verify that schema contract via the decorator entry point. - -- [ ] **Step 1: Create the test file with helpers and the first two tests** - - Create `tests/test_core/function_pod/test_function_pod_ctx_arg.py`: - - ```python - # tests/test_core/function_pod/test_function_pod_ctx_arg.py - """Tests for the ctx_arg parameter on the @function_pod decorator. - - Full parity with tests/test_core/side_effect_function/test_side_effect_function_pod.py, - exercising @function_pod(ctx_arg=...) as the preferred side-effect pod entry point. - """ - from __future__ import annotations - - import pytest - import pyarrow as pa - - from orcapod.core.streams import ArrowTableStream - from orcapod.side_effects import InvocationContext - - - def _make_stream(n: int = 3) -> ArrowTableStream: - """Simple stream: tag=id (int), data=value (int).""" - schema = pa.schema([ - pa.field("id", pa.int64(), nullable=False), - pa.field("value", pa.int64(), nullable=False), - ]) - table = pa.table( - {"id": list(range(n)), "value": list(range(n))}, - schema=schema, - ) - return ArrowTableStream(table, tag_columns=["id"]) - - - def _make_pipeline_db(): - """Return a fresh in-memory ArrowDatabase.""" - from orcapod.databases.in_memory_databases import InMemoryArrowDatabase - return InMemoryArrowDatabase() - - - class TestCtxArgSchema: - """Scenarios 1–2: schema inference and ctx stripping via @function_pod(ctx_arg=...).""" - - def test_ctx01_ctx_stripped_from_input_schema(self): - """Scenario 1: ctx param absent from pod.input_data_schema; present in data function.""" - from orcapod.core.function_pod import function_pod - - @function_pod(ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"result_{value}" - - pod = my_fn.pod - - # Pod's exposed input schema excludes 'ctx' - assert "ctx" not in pod.input_data_schema - assert "value" in pod.input_data_schema - assert pod.input_data_schema["value"] == int - - # Underlying data function retains full schema (ctx included for hashing) - assert "ctx" in pod._data_function.input_data_schema - - # Output schema has the inferred key (function return type → str) - assert "result" in pod._data_function.output_data_schema - assert pod._data_function.output_data_schema["result"] == str - - def test_ctx02_ctx_arg_name_property(self): - """Scenario 2: pod.ctx_arg_name reflects the configured arg name.""" - from orcapod.core.function_pod import function_pod - - @function_pod(ctx_arg="invocation_ctx") - def my_fn(value: int, invocation_ctx: InvocationContext) -> str: - return f"r_{value}" - - pod = my_fn.pod - assert pod.ctx_arg_name == "invocation_ctx" - assert "invocation_ctx" not in pod.input_data_schema - assert "value" in pod.input_data_schema - ``` - -- [ ] **Step 2: Run these two tests** - - ```bash - uv run pytest tests/test_core/function_pod/test_function_pod_ctx_arg.py::TestCtxArgSchema -v - ``` - Expected: both `PASSED` - ---- - -## Task 5: Add execution scenarios (3–4) — sync and async standalone - -**Files:** -- Modify: `tests/test_core/function_pod/test_function_pod_ctx_arg.py` - -**Context:** Scenarios 3–4 verify that executing a `@function_pod(ctx_arg=...)` pod via -`pod.process(stream).iter_data()` correctly injects `InvocationContext` per row. Scenario 3 -is synchronous, scenario 4 uses an async user function (the sync wrapper handles it). - -- [ ] **Step 1: Append the execution test class to the file** - - ```python - class TestCtxArgStandaloneExecution: - """Scenarios 3–4: standalone execution via pod.process() / FunctionPodStream.""" - - def test_ctx03_sync_execution_injects_ctx_per_row(self): - """Scenario 3: ctx injected per row; output values correct; tags pass through.""" - from orcapod.core.function_pod import function_pod - - received_ctx: list[InvocationContext] = [] - - @function_pod(ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - received_ctx.append(ctx) - return f"v{value}" - - pod = my_fn.pod - stream = _make_stream(3) - rows = list(pod.process(stream).iter_data()) - - assert len(rows) == 3 - assert len(received_ctx) == 3 - - # Each ctx is a proper InvocationContext - for ctx in received_ctx: - assert isinstance(ctx, InvocationContext) - assert isinstance(ctx.invocation_hash, str) - assert len(ctx.invocation_hash) > 0 - - # Output values match expected - for i, (tag, data) in enumerate(rows): - assert data.as_dict()["result"] == f"v{i}" - - # Tags pass through unchanged - assert rows[0][0].as_dict()["id"] == 0 - assert rows[2][0].as_dict()["id"] == 2 - - def test_ctx04_async_fn_routed_through_sync_path(self): - """Scenario 4: async user function runs correctly via synchronous wrapper.""" - import asyncio - from orcapod.core.function_pod import function_pod - - @function_pod(ctx_arg="ctx") - async def my_async_fn(value: int, ctx: InvocationContext) -> str: - await asyncio.sleep(0) - return f"async_{value}" - - pod = my_async_fn.pod - stream = _make_stream(2) - rows = list(pod.process(stream).iter_data()) - - assert len(rows) == 2 - assert rows[0][1].as_dict()["result"] == "async_0" - assert rows[1][1].as_dict()["result"] == "async_1" - ``` - -- [ ] **Step 2: Run scenarios 1–4** - - ```bash - uv run pytest tests/test_core/function_pod/test_function_pod_ctx_arg.py -v - ``` - Expected: all 4 pass - ---- - -## Task 6: Add DB-backed and pipeline scenarios (5–6) - -**Files:** -- Modify: `tests/test_core/function_pod/test_function_pod_ctx_arg.py` - -**Context:** Scenarios 5–6 verify that the `@function_pod(ctx_arg=...)` pod integrates with -DB-backed execution via `FunctionJobNode` and `PipelineJob`. These mirror SF-06 and SF-12 -from the `side_effect_function_pod` suite. - -- [ ] **Step 1: Append the DB-backed test class** - - ```python - class TestCtxArgDBBacked: - """Scenarios 5–6: DB-backed execution via FunctionJobNode and PipelineJob.""" - - def test_ctx05_function_job_node_caches_output(self): - """Scenario 5: second run uses cached result; fn not called again.""" - from orcapod.core.function_pod import function_pod - from orcapod.core.nodes.function_node import FunctionJobNode - - call_count = 0 - - @function_pod(ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - return f"r{value}" - - pod = my_fn.pod - stream = _make_stream(2) - pipeline_db = _make_pipeline_db() - - node1 = FunctionJobNode(function_pod=pod, input_stream=stream) - node1.attach_databases(pipeline_database=pipeline_db) - results1 = node1.execute(stream) - assert len(results1) == 2 - assert call_count == 2 - - # Second run — same pod, same data, same DB — must NOT call fn again - node2 = FunctionJobNode(function_pod=pod, input_stream=stream) - node2.attach_databases(pipeline_database=pipeline_db) - results2 = node2.execute(stream) - assert len(results2) == 2 - assert call_count == 2 # NOT incremented - - # Both runs produce identical result values - for (_, d1), (_, d2) in zip(results1, results2): - assert d1.as_dict()["result"] == d2.as_dict()["result"] - - def test_ctx06_pipeline_job_integration(self): - """Scenario 6: end-to-end PipelineJob compilation + execution with ctx_arg pod.""" - from orcapod.core.function_pod import function_pod - from orcapod.pipeline.job import PipelineJob - from orcapod.core.sources.dict_source import DictSource - - received_ctx: list[InvocationContext] = [] - - @function_pod(ctx_arg="ctx") - def transform(value: int, ctx: InvocationContext) -> str: - received_ctx.append(ctx) - return f"result_{value}" - - pod = transform.pod - db = _make_pipeline_db() - - with PipelineJob(name="test_ctx_pod", store=db) as job: - source = DictSource( - [{"id": 0, "value": 10}, {"id": 1, "value": 20}], - tag_columns=["id"], - ) - pod.process(source) - - job.run() - - # fn called once per row, ctx injected each time - assert len(received_ctx) == 2 - for ctx in received_ctx: - assert isinstance(ctx, InvocationContext) - ``` - -- [ ] **Step 2: Run scenarios 1–6** - - ```bash - uv run pytest tests/test_core/function_pod/test_function_pod_ctx_arg.py -v - ``` - Expected: all 6 pass - ---- - -## Task 7: Add decorator and collision scenarios (7–9) - -**Files:** -- Modify: `tests/test_core/function_pod/test_function_pod_ctx_arg.py` - -**Context:** Scenarios 7–8 verify the two decorator forms (`ctx_arg` only, and `ctx_arg` with -explicit `output_keys`). Scenario 9 verifies that a `ctx_arg` name that collides with a data -column in the input stream raises `ValueError` at process time. - -- [ ] **Step 1: Append the decorator and collision test classes** - - ```python - class TestCtxArgDecoratorForms: - """Scenarios 7–8: decorator usage with ctx_arg.""" - - def test_ctx07_decorator_ctx_arg_only(self): - """Scenario 7: @function_pod(ctx_arg='ctx') — output_keys inferred from return type.""" - from orcapod.core.function_pod import function_pod, FunctionPod - - @function_pod(ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"v{value}" - - # my_fn is still callable (wraps original fn) - assert callable(my_fn) - # .pod is attached - pod = my_fn.pod - assert isinstance(pod, FunctionPod) - assert pod.ctx_arg_name == "ctx" - # Output key inferred from return annotation - assert "result" in pod._data_function.output_data_schema - - def test_ctx08_decorator_with_explicit_output_keys(self): - """Scenario 8: @function_pod(output_keys=['out'], ctx_arg='ctx') — explicit output_keys.""" - from orcapod.core.function_pod import function_pod, FunctionPod - - @function_pod(output_keys=["out"], ctx_arg="ctx") - def my_fn(value: int, ctx: InvocationContext) -> str: - return f"v{value}" - - pod = my_fn.pod - assert isinstance(pod, FunctionPod) - assert pod.ctx_arg_name == "ctx" - assert "out" in pod._data_function.output_data_schema - assert "ctx" not in pod.input_data_schema - - - class TestCtxArgCollision: - """Scenario 9: ctx_arg names a param not in the function — ValueError at decoration time.""" - - def test_ctx09_invalid_ctx_arg_raises_at_decoration(self): - """Scenario 9: ValueError at decoration time if ctx_arg is not a parameter of the function. - - ``_FunctionPodBase.__init__`` raises immediately when ``ctx_arg_name`` is not - in the underlying data function's input schema. - """ - from orcapod.core.function_pod import function_pod - - with pytest.raises(ValueError, match="ctx_arg_name"): - @function_pod(ctx_arg="nonexistent_param") - def my_fn(value: int) -> str: - return str(value) - ``` - -- [ ] **Step 2: Run scenarios 1–9** - - ```bash - uv run pytest tests/test_core/function_pod/test_function_pod_ctx_arg.py -v - ``` - Expected: all 9 pass - ---- - -## Task 8: Add cached pod wrapping scenario (10) - -**Files:** -- Modify: `tests/test_core/function_pod/test_function_pod_ctx_arg.py` - -**Context:** Scenario 10 verifies that wrapping a `ctx_arg` pod in `CachedFunctionPod` (via -`pod_cache_database=`) does not break context injection. On a cache miss the inner pod's -`process_data` runs and injects `InvocationContext`. On a cache hit the cached output is -returned directly (fn not called). The wrapped pod's `input_data_schema` must still exclude ctx. - -- [ ] **Step 1: Append the cached pod test class** - - ```python - class TestCtxArgCachedPodWrapping: - """Scenario 10: @function_pod(ctx_arg=..., pod_cache_database=...) — CachedFunctionPod wrapping.""" - - def test_ctx10_cached_pod_injects_ctx_on_miss_skips_on_hit(self): - """Scenario 10: CachedFunctionPod wrapping a ctx_arg pod: - - ctx stripped from exposed schema - - InvocationContext injected on cache miss - - fn NOT called on cache hit; output same - """ - from orcapod.core.function_pod import function_pod - from orcapod.core.cached_function_pod import CachedFunctionPod - - received_ctx: list[InvocationContext] = [] - call_count = 0 - - db = _make_pipeline_db() - - @function_pod(ctx_arg="ctx", pod_cache_database=db) - def my_fn(value: int, ctx: InvocationContext) -> str: - nonlocal call_count - call_count += 1 - received_ctx.append(ctx) - return f"v{value}" - - pod = my_fn.pod - assert isinstance(pod, CachedFunctionPod) - - # Schema: ctx stripped from wrapped pod's exposed input schema - assert "ctx" not in pod.input_data_schema - assert "value" in pod.input_data_schema - assert pod.ctx_arg_name == "ctx" - - stream = _make_stream(2) - - # First run: cache miss → fn called, InvocationContext injected - rows1 = list(pod.process(stream).iter_data()) - assert call_count == 2 - assert len(received_ctx) == 2 - for ctx in received_ctx: - assert isinstance(ctx, InvocationContext) - assert len(ctx.invocation_hash) > 0 - - # Second run: cache hit → fn NOT called - rows2 = list(pod.process(stream).iter_data()) - assert call_count == 2 # unchanged — cache hit - - # Output values must match - for (_, d1), (_, d2) in zip(rows1, rows2): - assert d1.as_dict()["result"] == d2.as_dict()["result"] - ``` - -- [ ] **Step 2: Run all 10 scenarios** - - ```bash - uv run pytest tests/test_core/function_pod/test_function_pod_ctx_arg.py -v - ``` - Expected: all 10 pass - -- [ ] **Step 3: Run the full test suite to check for regressions** - - ```bash - uv run pytest tests/ -v --tb=short -q - ``` - Expected: all pass (or at minimum: no new failures beyond pre-existing ones) - -- [ ] **Step 4: Commit the new test file** - - ```bash - git add tests/test_core/function_pod/test_function_pod_ctx_arg.py - git commit -m "test(function-pod): add full ctx_arg test coverage via @function_pod decorator (ITL-544)" - ``` - ---- - -## Task 9: File follow-up Linear issue - -**Context:** Per the spec, a follow-up issue must be filed to track full removal of -`side_effect_function_pod` (export removal, decorator deletion, usage migration). -Use the Linear MCP tool. - -- [ ] **Step 1: File the follow-up issue** - - ``` - mcp__claude_ai_Linear__save_issue( - title: "Remove side_effect_function_pod decorator (post-ITL-544 cleanup)", - team: "Tools", - description: """ - ## Overview - - ``side_effect_function_pod`` is superseded by ``@function_pod(ctx_arg=...)`` (ITL-544). - This issue tracks full removal now that the preferred path is tested and documented. - - ## Goals & Success Criteria - - * ``side_effect_function_pod`` removed from ``src/orcapod/core/function_pod.py``. - * Removed from ``__init__.py`` exports. - * All existing usages (tests, docs, examples) migrated to ``@function_pod(ctx_arg=...)``. - * No references to ``side_effect_function_pod`` remain in the codebase. - - ## Scope & Boundaries - - In scope: - * Delete the ``side_effect_function_pod`` function. - * Update all call sites to the ``@function_pod(ctx_arg=...)`` form. - * Update ``tests/test_core/side_effect_function/test_side_effect_function_pod.py`` - to use the new decorator. - - Out of scope: - * Changes to ``FunctionPod.from_fn`` or any other API surface. - - ## Dependencies & Risks - - * Depends on ITL-544 being merged first. - """ - ) - ``` - -- [ ] **Step 2: Note the new issue ID for the PR description** - ---- - -## Task 10: Create the PR - -- [ ] **Step 1: Push the branch** - - ```bash - git push -u origin eywalker/itl-544-fix-quick-patches-ctx_arg-on-function-pod-empty-opts - ``` - -- [ ] **Step 2: Create the PR targeting `main`** - - ```bash - gh pr create \ - --base main \ - --title "fix: ctx_arg on function pod + empty-opts ray.remote (ITL-544)" \ - --body "$(cat <<'EOF' - ## Summary - - Closes ITL-544. - - Two quick patches promoted to properly tested, documented first-class capabilities: - - - **Ray executor fix:** Updated two mocks in `test_regression_fixes.py` to handle both `ray.remote(fn)` (empty-opts direct path) and `ray.remote(**opts)(fn)` (factory path). No changes to `ray.py` — the guard is correct. - - **`ctx_arg` test coverage:** Added `tests/test_core/function_pod/test_function_pod_ctx_arg.py` with 10 scenarios covering schema inference, sync/async standalone execution, DB-backed execution, full pipeline, decorator forms, ctx collision, and cached pod wrapping. - - **Docstring update:** `side_effect_function_pod` docstring now points to `@function_pod(ctx_arg=...)` as the preferred path. Follow-up issue filed for full removal. - - ## Test plan - - - [ ] `uv run pytest tests/test_core/test_regression_fixes.py -v` — all pass (including the two previously failing tests) - - [ ] `uv run pytest tests/test_core/function_pod/test_function_pod_ctx_arg.py -v` — all 10 new tests pass - - [ ] `uv run pytest tests/test_core/side_effect_function/ -v` — no regressions - - [ ] `uv run pytest tests/ -q` — full suite clean - - 🤖 Generated with [Claude Code](https://claude.com/claude-code) - EOF - )" - ``` diff --git a/superpowers/plans/2026-07-17-itl-535-schema-versioning-migration.md b/superpowers/plans/2026-07-17-itl-535-schema-versioning-migration.md deleted file mode 100644 index 27f64cd70..000000000 --- a/superpowers/plans/2026-07-17-itl-535-schema-versioning-migration.md +++ /dev/null @@ -1,2666 +0,0 @@ -# Schema Versioning + pdb/rdb v0→v1 Migration Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Introduce a first-class schema versioning framework for Orcapod's pipeline DB (pdb) and result DB (rdb), and provide a v0→v1 migration that converts ContentHash columns to binary, adds missing hash columns, and detects legacy tables with a hard stop. - -**Architecture:** Schema version is encoded in the storage path tuple (e.g., `node_identity_path + ("pdb_v1",)`), so different schema versions live at physically separate paths with no interference. `FunctionJobNode` and `ResultCache` detect v0 tables on first DB access and raise `SchemaVersionError` unless the node has explicitly opted in via `NodeConfig.ignore_schema`. The migration package reads from v0, transforms, and writes to v1 paths using `skip_duplicates=True` for idempotency and concurrency safety. - -**Tech Stack:** Python 3.11+, PyArrow, Delta Lake (`deltalake` 1.0.2), Typer (CLI), pytest + `uv run` - ---- - -## File Map - -**Created:** -- `src/orcapod/migrations/__init__.py` — public migration API -- `src/orcapod/migrations/types.py` — `MigrationResult` dataclass -- `src/orcapod/migrations/pipeline_db.py` — `migrate_pipeline_v0_to_v1()`, `migrate_node()` -- `src/orcapod/migrations/result_db.py` — `migrate_result_v0_to_v1()` -- `src/orcapod/cli/migrate.py` — `orcapod migrate pipeline-db` / `result-db` sub-commands -- `tests/test_migrations/__init__.py` -- `tests/test_migrations/test_migration_types.py` -- `tests/test_migrations/test_result_db.py` -- `tests/test_migrations/test_pipeline_db.py` -- `tests/test_migrations/test_golden.py` -- `tests/test_cli/test_migrate.py` -- `tests/fixtures/` — committed sample Delta Lake tables for each schema version - -**Modified:** -- `src/orcapod/types.py` — add `ContentHash.from_prefixed_digest()`, `NodeConfig.ignore_schema` -- `src/orcapod/errors.py` — add `SchemaVersionError` -- `src/orcapod/system_constants.py` — add `PIPELINE_DB_SCHEMA_VERSION`, `RESULT_DB_SCHEMA_VERSION` -- `src/orcapod/protocols/database_protocols.py` — add `table_exists()` to `ArrowDatabaseProtocol` -- `src/orcapod/databases/delta_lake_databases.py` — implement `table_exists()` -- `src/orcapod/databases/in_memory_databases.py` — implement `table_exists()` -- `src/orcapod/databases/noop_database.py` — implement `table_exists()` -- `src/orcapod/databases/connector_arrow_database.py` — implement `table_exists()` -- `src/orcapod/databases/extension_aware_database.py` — implement `table_exists()` -- `src/orcapod/core/nodes/function_node.py` — path versioning, schema detection, binary ContentHash, remove ITL-508 guard -- `src/orcapod/core/result_cache.py` — path versioning, schema detection, binary ContentHash -- `src/orcapod/core/cached_function_pod.py` — binary variation hash serialization -- `src/orcapod/cli/__init__.py` — register `migrate` sub-group - ---- - -## Task 1: `ContentHash.from_prefixed_digest()` classmethod - -**Files:** -- Modify: `src/orcapod/types.py` (after `to_prefixed_digest()`, around line 648) -- Test: `tests/test_types.py` - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_types.py`: - -```python -class TestContentHashPrefixedDigest: - """Tests for ContentHash.from_prefixed_digest() round-trip.""" - - def test_roundtrip_sha256(self): - h = ContentHash("sha256", bytes(range(32))) - assert ContentHash.from_prefixed_digest(h.to_prefixed_digest()) == h - - def test_roundtrip_arrow_method(self): - h = ContentHash("arrow_v2.1", b"\xde\xad\xbe\xef" * 8) - result = ContentHash.from_prefixed_digest(h.to_prefixed_digest()) - assert result.method == "arrow_v2.1" - assert result.digest == b"\xde\xad\xbe\xef" * 8 - - def test_preserves_binary_digest_with_colon_bytes(self): - # digest bytes that themselves contain a colon — only the first colon splits - digest = b"abc:def" - h = ContentHash("sha256", digest) - assert ContentHash.from_prefixed_digest(h.to_prefixed_digest()) == h - - def test_inverse_of_to_prefixed_digest(self): - """from_prefixed_digest is the exact inverse of to_prefixed_digest.""" - for method, digest in [ - ("sha256", b"\x00" * 32), - ("md5", b"\xff" * 16), - ("arrow_v2.1", b"\x01\x02\x03"), - ]: - h = ContentHash(method, digest) - assert ContentHash.from_prefixed_digest(h.to_prefixed_digest()) == h -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_types.py::TestContentHashPrefixedDigest -v -``` -Expected: `AttributeError: type object 'ContentHash' has no attribute 'from_prefixed_digest'` - -- [ ] **Step 3: Implement the classmethod** - -In `src/orcapod/types.py`, insert after `to_prefixed_digest()` (after line 648): - -```python - @classmethod - def from_prefixed_digest(cls, data: bytes) -> "ContentHash": - """Parse method-prefixed raw bytes back into a ``ContentHash``. - - Inverse of ``to_prefixed_digest()``. - - Args: - data: Bytes in the form ``b"{method}:{raw_digest}"``, as - produced by ``to_prefixed_digest()``. - - Returns: - A new ``ContentHash`` instance. - - Raises: - ValueError: If ``data`` does not contain a colon separator. - """ - colon_idx = data.index(b":") - method = data[:colon_idx].decode("ascii") - digest = data[colon_idx + 1:] - return cls(method=method, digest=digest) -``` - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/test_types.py::TestContentHashPrefixedDigest -v -``` -Expected: 4 PASSED - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/types.py tests/test_types.py -git commit -m "feat(types): add ContentHash.from_prefixed_digest() classmethod" -``` - ---- - -## Task 2: `SchemaVersionError` + `NodeConfig.ignore_schema` - -**Files:** -- Modify: `src/orcapod/errors.py` -- Modify: `src/orcapod/types.py` -- Test: `tests/test_types.py` (extend `TestNodeConfig`) - -- [ ] **Step 1: Write the failing tests** - -Add to `tests/test_types.py` in `TestNodeConfig`: - -```python - def test_ignore_schema_defaults_to_none(self): - config = NodeConfig() - assert config.ignore_schema is None - - def test_ignore_schema_set(self): - config = NodeConfig(ignore_schema=("v0",)) - assert config.ignore_schema == ("v0",) - - def test_merge_ignore_schema_none_in_other_self_wins(self): - base = NodeConfig(ignore_schema=("v0",)) - other = NodeConfig() # ignore_schema=None - result = base.merge(other) - assert result.ignore_schema == ("v0",) - - def test_merge_ignore_schema_other_overrides(self): - base = NodeConfig(ignore_schema=("v0",)) - other = NodeConfig(ignore_schema=("v0", "v1")) - result = base.merge(other) - assert result.ignore_schema == ("v0", "v1") - - def test_merge_ignore_schema_empty_tuple_overrides(self): - """Empty tuple () is a valid explicit value that overrides None.""" - base = NodeConfig(ignore_schema=("v0",)) - other = NodeConfig(ignore_schema=()) - result = base.merge(other) - assert result.ignore_schema == () -``` - -Also add to `tests/test_types.py` (new class): - -```python -class TestSchemaVersionError: - def test_is_exception(self): - from orcapod.errors import SchemaVersionError - err = SchemaVersionError("test message") - assert isinstance(err, Exception) - - def test_message_preserved(self): - from orcapod.errors import SchemaVersionError - err = SchemaVersionError("Pipeline DB at v0 path") - assert "v0" in str(err) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_types.py::TestNodeConfig::test_ignore_schema_defaults_to_none tests/test_types.py::TestSchemaVersionError -v -``` -Expected: AttributeError on `ignore_schema`, ImportError on `SchemaVersionError` - -- [ ] **Step 3: Add `SchemaVersionError` to `errors.py`** - -Append to `src/orcapod/errors.py`. Note: `errors.py` has no `__all__`, so no export list needs updating. - -```python -class SchemaVersionError(Exception): - """Raised when an old schema version is detected at a database path. - - Occurs when a table exists at the unversioned (v0) path but not at the - expected versioned path, and the node has not opted in to tolerating - the old schema via ``NodeConfig.ignore_schema``. - - To suppress this error, set ``node.node_config = NodeConfig(ignore_schema=("v0",))`` - and re-run. The node will recompute all results from scratch rather than - reading from the v0 cache. - """ -``` - -- [ ] **Step 4: Add `ignore_schema` to `NodeConfig` in `types.py`** - -Find the `NodeConfig` dataclass (around line 347). Replace the existing class with: - -```python -@dataclass(frozen=True, slots=True) -class NodeConfig: - """Per-node pipeline execution configuration. - - Attributes: - is_result_ephemeral: ``None`` inherits the default (``False``). - ``True`` writes new computation results to the pipeline-scoped - ephemeral store instead of the persistent result database. - Persistent cache hits are still served when available. Raises - ``RuntimeError`` at execution time if ``True`` but no ephemeral - store has been injected via ``set_ephemeral_store()``. - ignore_schema: Tuple of schema version strings that this node will - tolerate without raising ``SchemaVersionError``. ``None`` (default) - means no old schema is tolerated — any detected v0 table raises - ``SchemaVersionError``. Pass ``("v0",)`` to suppress the error - and allow the node to recompute all results from scratch. - """ - - is_result_ephemeral: bool | None = None - ignore_schema: tuple[str, ...] | None = None - - def merge(self, other: "NodeConfig") -> "NodeConfig": - """Return a new ``NodeConfig`` with ``other``'s non-``None`` fields overriding self. - - ``None`` fields in ``other`` are treated as "not set" and leave - self's value unchanged. - - Args: - other: The ``NodeConfig`` whose non-``None`` fields take precedence. - - Returns: - A new immutable ``NodeConfig``. - - Example: - NodeConfig(is_result_ephemeral=True).merge(NodeConfig()) - # → NodeConfig(is_result_ephemeral=True) - - NodeConfig(is_result_ephemeral=True).merge(NodeConfig(is_result_ephemeral=False)) - # → NodeConfig(is_result_ephemeral=False) - """ - return NodeConfig( - is_result_ephemeral=( - other.is_result_ephemeral - if other.is_result_ephemeral is not None - else self.is_result_ephemeral - ), - ignore_schema=( - other.ignore_schema - if other.ignore_schema is not None - else self.ignore_schema - ), - ) -``` - -- [ ] **Step 5: Run all tests to verify they pass** - -```bash -uv run pytest tests/test_types.py::TestNodeConfig tests/test_types.py::TestSchemaVersionError -v -``` -Expected: all PASSED - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/errors.py src/orcapod/types.py tests/test_types.py -git commit -m "feat(schema): add SchemaVersionError and NodeConfig.ignore_schema" -``` - ---- - -## Task 3: `ArrowDatabaseProtocol.table_exists()` + all backends - -**Files:** -- Modify: `src/orcapod/protocols/database_protocols.py` -- Modify: `src/orcapod/databases/delta_lake_databases.py` -- Modify: `src/orcapod/databases/in_memory_databases.py` -- Modify: `src/orcapod/databases/noop_database.py` -- Modify: `src/orcapod/databases/connector_arrow_database.py` -- Modify: `src/orcapod/databases/extension_aware_database.py` -- Test: `tests/test_databases/test_in_memory_database.py`, `tests/test_databases/test_delta_table_database.py`, `tests/test_databases/test_noop_database.py` - -- [ ] **Step 1: Write the failing tests** - -Add to `tests/test_databases/test_in_memory_database.py`: - -```python -class TestTableExists: - def test_returns_false_when_path_absent(self, db): - assert db.table_exists(("a", "b")) is False - - def test_returns_true_after_adding_record(self, db): - record = make_table(val=[1]) - db.add_record(("mypath",), b"id1", record) - assert db.table_exists(("mypath",)) is True - - def test_returns_true_before_flush(self, db): - """Pending (pre-flush) writes count as existing.""" - record = make_table(val=[1]) - db.add_record(("pending_path",), b"id1", record) - # InMemoryArrowDatabase doesn't require explicit flush, - # but verify pending_batches are seen. - assert db.table_exists(("pending_path",)) is True - - def test_scoped_db_sees_correct_path(self, db): - scoped = db.at("scope") - record = make_table(val=[1]) - scoped.add_record(("sub",), b"id1", record) - assert scoped.table_exists(("sub",)) is True - assert db.table_exists(("scope", "sub")) is True - assert db.table_exists(("sub",)) is False -``` - -Add to `tests/test_databases/test_delta_table_database.py`: - -```python -class TestTableExists: - def test_returns_false_when_path_absent(self, db): - assert db.table_exists(("no", "table", "here")) is False - - def test_returns_true_after_write_and_flush(self, db): - record = make_table(val=[1]) - db.add_record(("mypath",), b"id1", record) - db.flush() - assert db.table_exists(("mypath",)) is True - - def test_returns_false_for_different_path(self, db): - record = make_table(val=[1]) - db.add_record(("path_a",), b"id1", record) - db.flush() - assert db.table_exists(("path_b",)) is False -``` - -Add to `tests/test_databases/test_noop_database.py`: - -```python -class TestTableExists: - def test_always_returns_false(self): - from orcapod.databases.noop_database import NoOpArrowDatabase - db = NoOpArrowDatabase() - assert db.table_exists(("any", "path")) is False -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_databases/test_in_memory_database.py::TestTableExists tests/test_databases/test_delta_table_database.py::TestTableExists tests/test_databases/test_noop_database.py::TestTableExists -v -``` -Expected: `AttributeError: 'InMemoryArrowDatabase' object has no attribute 'table_exists'` - -- [ ] **Step 3: Add `table_exists` to `ArrowDatabaseProtocol`** - -In `src/orcapod/protocols/database_protocols.py`, add after `flush()` (around line 67), and add `"table_exists"` to the existing `__all__` list at the bottom of the file (around line 168): - -```python - def table_exists(self, record_path: tuple[str, ...]) -> bool: - """Return ``True`` if a table exists at the given path, even if it has no rows. - - For Delta Lake backends: checks whether the ``_delta_log/`` directory - exists at the resolved path. For in-memory backends: checks whether the - path key is present in the internal store (committed or pending). - - Args: - record_path: Path tuple identifying the table. - - Returns: - ``True`` if the table has been created; ``False`` if the path is - entirely absent. - """ - ... -``` - -- [ ] **Step 4: Implement in `DeltaTableDatabase`** - -In `src/orcapod/databases/delta_lake_databases.py`, add after `flush()`: - -```python - def table_exists(self, record_path: tuple[str, ...]) -> bool: - """Return ``True`` if a Delta Lake table exists at the given path. - - Args: - record_path: Path tuple identifying the table. - - Returns: - ``True`` if the table's ``_delta_log/`` directory is present on - the underlying storage; ``False`` otherwise. - """ - return self._get_delta_table(record_path) is not None -``` - -- [ ] **Step 5: Implement in `InMemoryArrowDatabase`** - -In `src/orcapod/databases/in_memory_databases.py`, add after `flush()`: - -```python - def table_exists(self, record_path: tuple[str, ...]) -> bool: - """Return ``True`` if any records exist at the given path (committed or pending). - - Args: - record_path: Path tuple identifying the table. - - Returns: - ``True`` if the path key is present in committed or pending store. - """ - record_key = "/".join(self._path_prefix + record_path) - return record_key in self._tables or record_key in self._pending_batches -``` - -- [ ] **Step 6: Implement in `NoOpArrowDatabase`** - -In `src/orcapod/databases/noop_database.py`, add after the existing `flush()` or similar no-op method: - -```python - def table_exists(self, record_path: tuple[str, ...]) -> bool: - """Always returns ``False`` — the no-op database never stores anything. - - Args: - record_path: Path tuple identifying the table. - - Returns: - Always ``False``. - """ - return False -``` - -- [ ] **Step 7: Implement in `ConnectorArrowDatabase`** - -In `src/orcapod/databases/connector_arrow_database.py`, add after `flush()`: - -```python - def table_exists(self, record_path: tuple[str, ...]) -> bool: - """Return ``True`` if the SQL table backing this path exists in the connector. - - Checks both committed (connector table) and pending (in-memory batch). - - Args: - record_path: Path tuple identifying the table. - - Returns: - ``True`` if the table exists in the connector or has pending writes. - """ - record_key = self._get_record_key(record_path) - if record_key in self._pending_batches: - return True - table_name = self._path_to_table_name(self._path_prefix + record_path) - return table_name in self._connector.get_table_names() -``` - -- [ ] **Step 8: Implement in `ExtensionAwareDatabase`** - -`ExtensionAwareDatabase` has no `_path_prefix` — it simply wraps `self._db` and passes all paths through unchanged. In `src/orcapod/databases/extension_aware_database.py`, add after the `flush()` delegation: - -```python - def table_exists(self, record_path: tuple[str, ...]) -> bool: - """Delegate to the underlying database's ``table_exists()``. - - ``ExtensionAwareDatabase`` does not modify paths, so the call is - passed through unchanged to the wrapped backend. - - Args: - record_path: Path tuple identifying the table. - - Returns: - Result of the underlying database's ``table_exists()`` call. - """ - return self._db.table_exists(record_path) -``` - -- [ ] **Step 9: Run all tests to verify they pass** - -```bash -uv run pytest tests/test_databases/test_in_memory_database.py::TestTableExists tests/test_databases/test_delta_table_database.py::TestTableExists tests/test_databases/test_noop_database.py::TestTableExists -v -``` -Expected: all PASSED - -- [ ] **Step 10: Run the full database test suite to check for regressions** - -```bash -uv run pytest tests/test_databases/ -v --tb=short -``` -Expected: all PASSED - -- [ ] **Step 11: Commit** - -```bash -git add src/orcapod/protocols/database_protocols.py \ - src/orcapod/databases/delta_lake_databases.py \ - src/orcapod/databases/in_memory_databases.py \ - src/orcapod/databases/noop_database.py \ - src/orcapod/databases/connector_arrow_database.py \ - src/orcapod/databases/extension_aware_database.py \ - tests/test_databases/test_in_memory_database.py \ - tests/test_databases/test_delta_table_database.py \ - tests/test_databases/test_noop_database.py -git commit -m "feat(db): add table_exists() to ArrowDatabaseProtocol and all backends" -``` - ---- - -## Task 4: Schema version constants - -**Files:** -- Modify: `src/orcapod/system_constants.py` - -- [ ] **Step 1: Add the constants** - -In `src/orcapod/system_constants.py`, add after the existing module-level constants (after line 23, before the `SystemConstant` class): - -```python -PIPELINE_DB_SCHEMA_VERSION = "pdb_v1" -RESULT_DB_SCHEMA_VERSION = "rdb_v1" -``` - -- [ ] **Step 2: Verify the constants import** - -```bash -uv run python -c "from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION, RESULT_DB_SCHEMA_VERSION; print(PIPELINE_DB_SCHEMA_VERSION, RESULT_DB_SCHEMA_VERSION)" -``` -Expected: `pdb_v1 rdb_v1` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/system_constants.py -git commit -m "feat(schema): add PIPELINE_DB_SCHEMA_VERSION and RESULT_DB_SCHEMA_VERSION constants" -``` - ---- - -## Task 5: `FunctionJobNode` — path versioning, schema detection, binary ContentHash, remove ITL-508 guard - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` -- Test: `tests/test_core/nodes/test_function_node_iteration.py` (extend) and new `tests/test_core/nodes/test_schema_detection.py` - -- [ ] **Step 1: Write failing tests for schema detection** - -Create `tests/test_core/nodes/test_schema_detection.py`: - -```python -"""Tests for FunctionJobNode old-schema detection and hard stop.""" -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.nodes.function_node import FunctionJobNode, _checked_pdb_paths -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.errors import SchemaVersionError -from orcapod.types import NodeConfig -from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION - - -def _make_node(db: InMemoryArrowDatabase) -> FunctionJobNode: - def double(x: int) -> int: - return x * 2 - - source = ArrowTableSource( - pa.table({"id": [1, 2], "x": [10, 20]}), tag_columns=["id"] - ) - pf = PythonDataFunction(double, output_keys="result") - from orcapod.core.function_pod import FunctionPod - pod = FunctionPod(source, pf) - return FunctionJobNode(pod, pipeline_database=db) - - -@pytest.fixture(autouse=True) -def clear_checked_paths(): - """Clear the process-level cache between tests.""" - _checked_pdb_paths.clear() - yield - _checked_pdb_paths.clear() - - -class TestPdbSchemaDetection: - def test_no_error_when_v1_table_exists(self, tmp_path): - """v1 table present → schema check passes immediately.""" - db = InMemoryArrowDatabase() - node = _make_node(db) - v1_path = node.node_identity_path + (PIPELINE_DB_SCHEMA_VERSION,) - # Seed the v1 path so it appears to exist - db.add_record(v1_path, b"\x00" * 16, pa.table({"x": [1]})) - # iter_data should not raise - list(node.iter_data()) # no exception - - def test_no_error_when_both_paths_absent(self, tmp_path): - """Fresh database (neither v0 nor v1 exists) → no error.""" - db = InMemoryArrowDatabase() - node = _make_node(db) - list(node.iter_data()) # no exception - - def test_raises_schema_version_error_when_v0_exists(self): - """v1 absent, v0 has records, no ignore_schema → SchemaVersionError.""" - db = InMemoryArrowDatabase() - node = _make_node(db) - v0_path = node.node_identity_path - # Seed the v0 path - db.add_record(v0_path, b"\x00" * 16, pa.table({"x": [1]})) - with pytest.raises(SchemaVersionError, match="v0"): - list(node.iter_data()) - - def test_no_error_when_v0_exists_and_ignored(self): - """v1 absent, v0 has records, ignore_schema=('v0',) → proceeds.""" - db = InMemoryArrowDatabase() - node = _make_node(db) - node.node_config = NodeConfig(ignore_schema=("v0",)) - v0_path = node.node_identity_path - db.add_record(v0_path, b"\x00" * 16, pa.table({"x": [1]})) - list(node.iter_data()) # no exception - - def test_cache_hit_skips_db_call(self, mocker): - """Second access to same v1 path skips table_exists entirely.""" - db = InMemoryArrowDatabase() - node = _make_node(db) - # Pre-populate the cache as if the check already ran - v1_path = node.node_identity_path + (PIPELINE_DB_SCHEMA_VERSION,) - _checked_pdb_paths.add(v1_path) - spy = mocker.spy(db, "table_exists") - list(node.iter_data()) - spy.assert_not_called() -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_core/nodes/test_schema_detection.py -v -``` -Expected: `ImportError: cannot import name '_checked_pdb_paths'` or `AttributeError` - -- [ ] **Step 3: Add imports and module-level cache to `function_node.py`** - -At the top of `src/orcapod/core/nodes/function_node.py`, add to the imports section: - -```python -from orcapod.errors import SchemaVersionError -from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION -``` - -Just before the class definition (after existing module-level constants), add: - -```python -# Process-level cache of v1 pipeline DB paths that have already been checked. -# Once a path is in this set, no further DB call is issued for it. -_checked_pdb_paths: set[tuple[str, ...]] = set() -``` - -- [ ] **Step 4: Add `_versioned_pipeline_path` property and `_ensure_pdb_schema()` method** - -In `FunctionJobNode`, after the `node_identity_path` property (around line 315), add: - -```python - @property - def _versioned_pipeline_path(self) -> tuple[str, ...]: - """Pipeline DB path with the current schema version suffix appended. - - This is the path used for all actual pipeline DB reads and writes. - ``node_identity_path`` (without the suffix) is retained for observer - contextualization and error messages only. - """ - return self.node_identity_path + (PIPELINE_DB_SCHEMA_VERSION,) - - def _ensure_pdb_schema(self) -> None: - """Check for an old-schema (v0) pipeline DB on first access per path. - - Detection flow (runs at most once per v1 path per process): - - 1. If the v1 path is already in ``_checked_pdb_paths`` → return immediately. - 2. If the v1 table exists → mark checked and return. - 3. If the v0 path (bare ``node_identity_path``) has a table: - - ``"v0"`` in ``node_config.ignore_schema`` → log info, continue. - - Otherwise → raise ``SchemaVersionError``. - 4. Neither path exists → fresh database, continue. - 5. Mark v1 path as checked. - - Raises: - SchemaVersionError: If a v0 table is detected and not ignored. - """ - if self._pipeline_database is None: - return - v1_path = self._versioned_pipeline_path - if v1_path in _checked_pdb_paths: - return - if self._pipeline_database.table_exists(v1_path): - _checked_pdb_paths.add(v1_path) - return - v0_path = self.node_identity_path - if self._pipeline_database.table_exists(v0_path): - _checked_pdb_paths.add(v1_path) - ignore = self._node_config.ignore_schema or () - if "v0" not in ignore: - raise SchemaVersionError( - f"Pipeline DB rows found at v0 schema path {v0_path!r}.\n" - "Run migration first:\n" - " orcapod migrate pipeline-db \n" - "To suppress this error and recompute all results instead, set:\n" - " node.node_config = NodeConfig(ignore_schema=(\"v0\",))" - ) - logger.info( - "Pipeline DB v0 schema detected at %r — proceeding because " - "ignore_schema=%r", - v0_path, - ignore, - ) - _checked_pdb_paths.add(v1_path) -``` - -- [ ] **Step 5: Update `_fetch_joined_records()` to call `_ensure_pdb_schema()` and use the versioned path** - -In `_fetch_joined_records()` (around line 1818), at the very start of the method body, add: - -```python - self._ensure_pdb_schema() -``` - -Then replace every occurrence of `self.node_identity_path` passed to `self._pipeline_database.*` calls with `self._versioned_pipeline_path`. These calls include: -- `self._pipeline_database.get_all_records(self.node_identity_path, ...)` -- `self._pipeline_database.get_records_by_ids(self.node_identity_path, ...)` -- `self._pipeline_database.get_records_with_column_value(self.node_identity_path, ...)` - -Do NOT replace occurrences used for observer contextualization or error messages (e.g., `obs.contextualize(*self.node_identity_path)`, `node_identity_path=self.node_identity_path`). - -- [ ] **Step 6: Update `add_pipeline_record()` — remove ITL-508 guard, use versioned path, binary ContentHash** - -In `add_pipeline_record()` (around line 1583): - -**a) Remove the ITL-508 guard** (lines ~1629–1644). Delete this block entirely: - -```python - # Guard against pre-ITL-508 pipeline DB records that are missing the new - # versioning columns. If such records exist, fail fast with a clear message - # ... - _all_existing = self._pipeline_database.get_all_records(self.node_identity_path) - if _all_existing is not None and _all_existing.num_rows > 0: - _missing = [...] - if _missing: - raise ValueError(...) -``` - -**b) Replace `self.node_identity_path` with `self._versioned_pipeline_path`** in all DB calls within this method: -- `get_records_with_column_value(self.node_identity_path, ...)` → `self._versioned_pipeline_path` -- `add_record(self.node_identity_path, ...)` → `self._versioned_pipeline_path` - -**c) Change ContentHash columns in `meta_table` from `large_string` to `large_binary`**: - -```python - meta_table = pa.table( - { - constants.DATA_RECORD_ID: pa.array( - [data_record_id.bytes], type=pa.large_binary() - ), - constants.NODE_CONTENT_HASH_COL: pa.array( - [self.content_hash().to_prefixed_digest()], type=pa.large_binary() - ), - constants.INPUT_DATA_HASH_COL: pa.array( - [input_data.content_hash().to_prefixed_digest()], type=pa.large_binary() - ), - constants.OUTPUT_DATA_HASH_COL: pa.array( - [output_data_hash.to_prefixed_digest() if output_data_hash is not None else None], - type=pa.large_binary(), - ), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array( - [input_data.data_context_key], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array( - [computed], type=pa.bool_() - ), - constants.IS_EPHEMERAL_COL: pa.array( - [is_ephemeral], type=pa.bool_() - ), - _PIPELINE_BASE_ENTRY_ID_COL: pa.array( - [base_entry_id], type=pa.large_binary() - ), - _PIPELINE_RECOMPUTATION_INDEX_COL: pa.array( - [new_index], type=pa.int32() - ), - } - ) -``` - -- [ ] **Step 7: Update `_fetch_joined_records()` to read ContentHash as binary** - -In `_fetch_joined_records()`, wherever `__node_content_hash`, `__input_data_hash`, or `__output_data_hash` are read from the pipeline DB table and passed as `ContentHash` objects, use `ContentHash.from_prefixed_digest(bytes_value)` instead of `ContentHash.from_string(string_value)`. - -Search for uses of `NODE_CONTENT_HASH_COL`, `INPUT_DATA_HASH_COL`, `OUTPUT_DATA_HASH_COL` in `_fetch_joined_records()` and update accordingly. The column values will now be `bytes` (PyArrow `large_binary`), not strings. - -- [ ] **Step 8: Run schema detection tests** - -```bash -uv run pytest tests/test_core/nodes/test_schema_detection.py -v -``` -Expected: all PASSED - -- [ ] **Step 9: Run the full FunctionJobNode test suite** - -```bash -uv run pytest tests/test_core/nodes/ -v --tb=short -``` -Expected: all PASSED (no regressions) - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py \ - tests/test_core/nodes/test_schema_detection.py -git commit -m "feat(schema): FunctionJobNode path versioning, binary ContentHash, schema detection" -``` - ---- - -## Task 6: `ResultCache` — path versioning, schema detection, binary ContentHash - -**Files:** -- Modify: `src/orcapod/core/result_cache.py` -- Test: `tests/test_core/test_result_cache.py` (extend) - -- [ ] **Step 1: Write failing tests for rdb schema detection** - -Add to `tests/test_core/test_result_cache.py`: - -```python -from orcapod.core.result_cache import _checked_rdb_paths -from orcapod.errors import SchemaVersionError -from orcapod.types import NodeConfig, ContentHash -from orcapod.system_constants import RESULT_DB_SCHEMA_VERSION - - -@pytest.fixture(autouse=True) -def clear_rdb_checked_paths(): - _checked_rdb_paths.clear() - yield - _checked_rdb_paths.clear() - - -class TestRdbSchemaDetection: - def _make_cache_with_db(self, db, record_path=("test",)): - return ResultCache( - result_database=db, - record_path=record_path, - ) - - def test_no_error_fresh_db(self): - db = InMemoryArrowDatabase() - cache = self._make_cache_with_db(db) - result = cache.lookup(make_input_data()) - assert result is None # no error - - def test_raises_when_v0_exists(self): - db = InMemoryArrowDatabase() - v0_path = ("test",) - db.add_record(v0_path, b"\x00" * 16, pa.table({"x": [1]})) - cache = self._make_cache_with_db(db, record_path=v0_path) - with pytest.raises(SchemaVersionError, match="v0"): - cache.lookup(make_input_data()) - - def test_no_error_when_ignored(self): - db = InMemoryArrowDatabase() - v0_path = ("test",) - db.add_record(v0_path, b"\x00" * 16, pa.table({"x": [1]})) - cache = self._make_cache_with_db(db, record_path=v0_path) - cache.set_ignore_schema(("v0",)) - result = cache.lookup(make_input_data()) - assert result is None - - def test_no_error_when_v1_exists(self): - db = InMemoryArrowDatabase() - v0_path = ("test",) - v1_path = v0_path + (RESULT_DB_SCHEMA_VERSION,) - # Seed v0 (would trigger error) AND v1 (should suppress check) - db.add_record(v0_path, b"\x00" * 16, pa.table({"x": [1]})) - db.add_record(v1_path, b"\x01" * 16, pa.table({"x": [2]})) - cache = self._make_cache_with_db(db, record_path=v0_path) - result = cache.lookup(make_input_data()) - assert result is None # v1 exists, no error -``` - -Note: `make_input_data()` is a helper returning a simple `Data` object — add it near the top of the test file: - -```python -def make_input_data(): - from orcapod.core.datagrams import Data - return Data(pa.table({"val": pa.array([42], type=pa.int64())})) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_core/test_result_cache.py::TestRdbSchemaDetection -v -``` -Expected: `ImportError: cannot import name '_checked_rdb_paths'` - -- [ ] **Step 3: Add imports and module-level cache + `set_ignore_schema` to `ResultCache`** - -At the top of `src/orcapod/core/result_cache.py`, add: - -```python -from orcapod.errors import SchemaVersionError -from orcapod.system_constants import RESULT_DB_SCHEMA_VERSION -``` - -After the imports (module level), add: - -```python -# Process-level cache of v1 result DB paths that have already been checked. -_checked_rdb_paths: set[tuple[str, ...]] = set() -``` - -In the `ResultCache` class, add `_ignore_schema` attribute and a setter: - -```python - def __init__( - self, - result_database: ArrowDatabaseProtocol, - record_path: tuple[str, ...], - auto_flush: bool = True, - ) -> None: - self._result_database = result_database - self._record_path = record_path - self._auto_flush = auto_flush - self._ignore_schema: tuple[str, ...] | None = None # NEW - - def set_ignore_schema(self, ignore_schema: tuple[str, ...] | None) -> None: - """Set which old schema versions to tolerate without raising ``SchemaVersionError``. - - Args: - ignore_schema: Tuple of schema version strings to tolerate (e.g. ``("v0",)``), - or ``None`` to use the default (raise on any old schema). - """ - self._ignore_schema = ignore_schema -``` - -- [ ] **Step 4: Add `_versioned_record_path` property and `_ensure_rdb_schema()` method** - -In `ResultCache`: - -```python - @property - def _versioned_record_path(self) -> tuple[str, ...]: - """Result DB path with the current schema version suffix appended.""" - return self._record_path + (RESULT_DB_SCHEMA_VERSION,) - - def _ensure_rdb_schema(self) -> None: - """Check for an old-schema (v0) result DB on first access per path. - - Same detection flow as ``FunctionJobNode._ensure_pdb_schema()``: - checks the v1 path first (cheap set lookup), then falls back to - calling ``table_exists()`` on both v1 and v0 paths. - - Raises: - SchemaVersionError: If a v0 table is detected and not ignored. - """ - v1_path = self._versioned_record_path - if v1_path in _checked_rdb_paths: - return - if self._result_database.table_exists(v1_path): - _checked_rdb_paths.add(v1_path) - return - v0_path = self._record_path - if self._result_database.table_exists(v0_path): - _checked_rdb_paths.add(v1_path) - ignore = self._ignore_schema or () - if "v0" not in ignore: - raise SchemaVersionError( - f"Result DB rows found at v0 schema path {v0_path!r}.\n" - "Run migration first:\n" - " orcapod migrate result-db \n" - "To suppress this error and recompute all results instead, set:\n" - " node.node_config = NodeConfig(ignore_schema=(\"v0\",))" - ) - logger.info( - "Result DB v0 schema detected at %r — proceeding because " - "ignore_schema=%r", - v0_path, - ignore, - ) - _checked_rdb_paths.add(v1_path) -``` - -- [ ] **Step 5: Update `lookup()` — call detection + use versioned path + binary hash** - -In `lookup()`: - -At the top: `self._ensure_rdb_schema()` - -Change the constraint dict to use binary: -```python - constraints: dict[str, Any] = { - constants.INPUT_DATA_HASH_COL: input_data.content_hash().to_prefixed_digest(), - } -``` - -Change `self._record_path` to `self._versioned_record_path` in the `get_records_with_column_value()` call. - -- [ ] **Step 6: Update `store()` — use versioned path + binary ContentHash columns** - -In `store()`: - -Change `self._record_path` to `self._versioned_record_path` in the `add_record()` call. - -Change the input hash column from `large_string` to `large_binary`: -```python - data_table = data_table.add_column( - 0, - constants.INPUT_DATA_HASH_COL, - pa.array( - [input_data.content_hash().to_prefixed_digest()], type=pa.large_binary() - ), - ) -``` - -After the variation columns are added (inside the `for name in var_table.column_names:` loop or after), convert the ContentHash variation columns to binary. Add this block after all variation columns are inserted: - -```python - # Convert ContentHash variation columns from string to binary (v1 schema). - _HASH_VAR_COLS = { - f"{constants.PF_VARIATION_PREFIX}function_signature_hash", - f"{constants.PF_VARIATION_PREFIX}function_content_hash", - } - for col_name in _HASH_VAR_COLS: - if col_name in data_table.column_names: - col_idx = data_table.column_names.index(col_name) - string_vals = data_table.column(col_name).to_pylist() - binary_vals = pa.array( - [ - ContentHash.from_string(s).to_prefixed_digest() if s is not None else None - for s in string_vals - ], - type=pa.large_binary(), - ) - data_table = data_table.set_column(col_idx, col_name, binary_vals) -``` - -Add the import at the top of `result_cache.py`: -```python -from orcapod.types import ContentHash -``` - -- [ ] **Step 7: Run schema detection and result cache tests** - -```bash -uv run pytest tests/test_core/test_result_cache.py -v --tb=short -``` -Expected: all PASSED - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/core/result_cache.py tests/test_core/test_result_cache.py -git commit -m "feat(schema): ResultCache path versioning, binary ContentHash, schema detection" -``` - ---- - -## Task 7: `CachedFunctionPod` — wire `ignore_schema` from `NodeConfig` to `ResultCache` - -**Files:** -- Modify: `src/orcapod/core/cached_function_pod.py` - -The `ignore_schema` setting lives on `FunctionJobNode.node_config`. `CachedFunctionPod` is constructed by `FunctionJobNode` and holds a `ResultCache`. When the node config changes (via `.node_config = NodeConfig(ignore_schema=...)`), the cache needs to receive the updated value. - -- [ ] **Step 1: Add `set_ignore_schema` call when `node_config` is set** - -In `src/orcapod/core/nodes/function_node.py`, update the `node_config` setter (around line 786): - -```python - @node_config.setter - def node_config(self, value: NodeConfig) -> None: - self._node_config = value - # Propagate ignore_schema to the result cache so it uses the same policy. - if self._cached_function_pod is not None: - self._cached_function_pod.set_ignore_schema(value.ignore_schema) - if self._ephemeral_cached_pod is not None: - self._ephemeral_cached_pod.set_ignore_schema(value.ignore_schema) -``` - -- [ ] **Step 2: Add `set_ignore_schema` method to `CachedFunctionPod`** - -In `src/orcapod/core/cached_function_pod.py`: - -```python - def set_ignore_schema(self, ignore_schema: tuple[str, ...] | None) -> None: - """Propagate ``ignore_schema`` setting to the underlying ``ResultCache``. - - Args: - ignore_schema: Tuple of schema version strings to tolerate, or ``None``. - """ - self._cache.set_ignore_schema(ignore_schema) -``` - -- [ ] **Step 3: Run integration test to confirm node_config propagates** - -```bash -uv run pytest tests/test_core/ -v --tb=short -q -``` -Expected: all PASSED - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/core/cached_function_pod.py src/orcapod/core/nodes/function_node.py -git commit -m "feat(schema): propagate NodeConfig.ignore_schema to ResultCache via CachedFunctionPod" -``` - ---- - -## Task 8: Migration package — `types.py` and `result_db.py` - -**Files:** -- Create: `src/orcapod/migrations/__init__.py` -- Create: `src/orcapod/migrations/types.py` -- Create: `src/orcapod/migrations/result_db.py` -- Create: `tests/test_migrations/__init__.py` -- Create: `tests/test_migrations/test_migration_types.py` -- Create: `tests/test_migrations/test_result_db.py` - -- [ ] **Step 1: Write failing tests for `MigrationResult` and `migrate_result_v0_to_v1()`** - -Create `tests/test_migrations/__init__.py` (empty). - -Create `tests/test_migrations/test_migration_types.py`: - -```python -"""Tests for MigrationResult dataclass.""" -from __future__ import annotations - -from orcapod.migrations.types import MigrationResult - - -class TestMigrationResult: - def test_fields(self): - r = MigrationResult( - rows_total=100, - rows_migrated=95, - rows_skipped=4, - rows_unresolvable=1, - elapsed_s=3.14, - dry_run=False, - ) - assert r.rows_total == 100 - assert r.rows_migrated == 95 - assert r.rows_skipped == 4 - assert r.rows_unresolvable == 1 - assert r.elapsed_s == 3.14 - assert r.dry_run is False - - def test_dry_run_field(self): - r = MigrationResult( - rows_total=10, - rows_migrated=0, - rows_skipped=0, - rows_unresolvable=0, - elapsed_s=0.1, - dry_run=True, - ) - assert r.dry_run is True -``` - -Create `tests/test_migrations/test_result_db.py`: - -```python -"""Tests for migrate_result_v0_to_v1().""" -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.databases import InMemoryArrowDatabase -from orcapod.migrations.result_db import migrate_result_v0_to_v1 -from orcapod.migrations.types import MigrationResult -from orcapod.system_constants import RESULT_DB_SCHEMA_VERSION -from orcapod.types import ContentHash - - -def _make_v0_rdb_row(input_hash: ContentHash, sig_hash: ContentHash, content_hash: ContentHash) -> pa.Table: - """Build a single v0 rdb row in the old large_string format.""" - return pa.table({ - "__input_data_hash": pa.array([input_hash.to_string()], type=pa.large_string()), - "__pf_var_function_name": pa.array(["my_func"], type=pa.large_string()), - "__pf_var_function_signature_hash": pa.array([sig_hash.to_string()], type=pa.large_string()), - "__pf_var_function_content_hash": pa.array([content_hash.to_string()], type=pa.large_string()), - "__pf_var_git_hash": pa.array(["abc123"], type=pa.large_string()), - "__pf_exec_executor_type": pa.array(["local"], type=pa.large_string()), - "__pf_exec_python_version": pa.array(["3.11"], type=pa.large_string()), - "__pod_ts": pa.array([0], type=pa.timestamp("us", tz="UTC")), - "result": pa.array([42], type=pa.int64()), - }) - - -_INPUT_HASH = ContentHash("sha256", bytes(range(32))) -_SIG_HASH = ContentHash("sha256", bytes(range(1, 33))) -_CONTENT_HASH = ContentHash("sha256", bytes(range(2, 34))) - - -class TestMigrateResultV0ToV1: - def test_happy_path_migrates_all_rows(self): - db = InMemoryArrowDatabase() - v0_path = ("mypod",) - row = _make_v0_rdb_row(_INPUT_HASH, _SIG_HASH, _CONTENT_HASH) - db.add_records(v0_path, row, skip_duplicates=False) - db.flush() - - result = migrate_result_v0_to_v1(db, v0_path, progress=False) - - assert result.rows_total == 1 - assert result.rows_migrated == 1 - assert result.rows_skipped == 0 - assert result.rows_unresolvable == 0 - assert result.dry_run is False - - def test_v1_row_has_binary_hash_columns(self): - db = InMemoryArrowDatabase() - v0_path = ("mypod",) - row = _make_v0_rdb_row(_INPUT_HASH, _SIG_HASH, _CONTENT_HASH) - db.add_records(v0_path, row, skip_duplicates=False) - db.flush() - - migrate_result_v0_to_v1(db, v0_path, progress=False) - - v1_path = v0_path + (RESULT_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - assert v1_table is not None - assert v1_table.schema.field("__input_data_hash").type == pa.large_binary() - assert v1_table.schema.field("__pf_var_function_signature_hash").type == pa.large_binary() - assert v1_table.schema.field("__pf_var_function_content_hash").type == pa.large_binary() - # git_hash stays as string - assert v1_table.schema.field("__pf_var_git_hash").type == pa.large_string() - - def test_v1_binary_values_decode_correctly(self): - db = InMemoryArrowDatabase() - v0_path = ("mypod",) - row = _make_v0_rdb_row(_INPUT_HASH, _SIG_HASH, _CONTENT_HASH) - db.add_records(v0_path, row, skip_duplicates=False) - db.flush() - - migrate_result_v0_to_v1(db, v0_path, progress=False) - - v1_path = v0_path + (RESULT_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - row_dict = v1_table.to_pylist()[0] - assert ContentHash.from_prefixed_digest(bytes(row_dict["__input_data_hash"])) == _INPUT_HASH - assert ContentHash.from_prefixed_digest(bytes(row_dict["__pf_var_function_signature_hash"])) == _SIG_HASH - assert ContentHash.from_prefixed_digest(bytes(row_dict["__pf_var_function_content_hash"])) == _CONTENT_HASH - - def test_idempotent_second_run_skips_all(self): - db = InMemoryArrowDatabase() - v0_path = ("mypod",) - row = _make_v0_rdb_row(_INPUT_HASH, _SIG_HASH, _CONTENT_HASH) - db.add_records(v0_path, row, skip_duplicates=False) - db.flush() - - migrate_result_v0_to_v1(db, v0_path, progress=False) - result2 = migrate_result_v0_to_v1(db, v0_path, progress=False) - - assert result2.rows_migrated == 0 - assert result2.rows_skipped == 1 - - def test_dry_run_writes_nothing(self): - db = InMemoryArrowDatabase() - v0_path = ("mypod",) - row = _make_v0_rdb_row(_INPUT_HASH, _SIG_HASH, _CONTENT_HASH) - db.add_records(v0_path, row, skip_duplicates=False) - db.flush() - - result = migrate_result_v0_to_v1(db, v0_path, dry_run=True, progress=False) - - v1_path = v0_path + (RESULT_DB_SCHEMA_VERSION,) - assert not db.table_exists(v1_path) - assert result.dry_run is True - assert result.rows_total == 1 - assert result.rows_migrated == 0 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_migrations/ -v -``` -Expected: `ModuleNotFoundError: No module named 'orcapod.migrations'` - -- [ ] **Step 3: Create `src/orcapod/migrations/types.py`** - -```python -"""Migration result types for Orcapod schema migrations.""" -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass -class MigrationResult: - """Summary of a completed schema migration run. - - Attributes: - rows_total: Total rows found at the v0 path. - rows_migrated: Rows successfully written to the v1 path. - rows_skipped: Rows already present at the v1 path (idempotent re-run). - rows_unresolvable: Rows whose result data was unreachable (e.g. ephemeral - result expired); ``__output_data_hash`` written as ``None`` for these. - elapsed_s: Wall-clock seconds elapsed during the migration. - dry_run: ``True`` if the run was a dry run (no writes performed). - """ - - rows_total: int - rows_migrated: int - rows_skipped: int - rows_unresolvable: int - elapsed_s: float - dry_run: bool -``` - -- [ ] **Step 4: Create `src/orcapod/migrations/result_db.py`** - -```python -"""v0 → v1 migration for the Orcapod result DB (rdb).""" -from __future__ import annotations - -import logging -import time -from typing import TYPE_CHECKING - -import pyarrow as pa - -from orcapod.migrations.types import MigrationResult -from orcapod.system_constants import RESULT_DB_SCHEMA_VERSION, constants -from orcapod.types import ContentHash - -if TYPE_CHECKING: - from orcapod.protocols.database_protocols import ArrowDatabaseProtocol - -logger = logging.getLogger(__name__) - -# rdb columns that hold orcapod-produced ContentHash values and must be -# converted from large_string (v0) to large_binary (v1). -_HASH_COLS = ( - constants.INPUT_DATA_HASH_COL, - f"{constants.PF_VARIATION_PREFIX}function_signature_hash", - f"{constants.PF_VARIATION_PREFIX}function_content_hash", -) - - -def migrate_result_v0_to_v1( - result_db: "ArrowDatabaseProtocol", - result_path: tuple[str, ...], - *, - dry_run: bool = False, - batch_size: int = 500, - progress: bool = True, -) -> MigrationResult: - """Migrate a result DB table from v0 schema to v1 schema. - - Reads records from ``result_path`` (v0, no suffix), converts all - orcapod-produced ``ContentHash`` columns from ``large_string`` to - ``large_binary`` (using ``ContentHash.to_prefixed_digest()``), and writes - the transformed rows to ``result_path + ("rdb_v1",)``. - - Rows already present at the v1 path are skipped (idempotent re-runs). - - Args: - result_db: The database containing the v0 table. - result_path: Bare v0 path tuple (no ``rdb_v1`` suffix). - dry_run: If ``True``, read and count rows but write nothing. - batch_size: Number of rows to process per batch. - progress: If ``True``, log progress at INFO level. - - Returns: - ``MigrationResult`` summarising the run. - """ - v1_path = result_path + (RESULT_DB_SCHEMA_VERSION,) - start = time.monotonic() - - v0_table = result_db.get_all_records(result_path) - if v0_table is None or v0_table.num_rows == 0: - logger.info("No v0 records found at %r — nothing to migrate.", result_path) - return MigrationResult( - rows_total=0, - rows_migrated=0, - rows_skipped=0, - rows_unresolvable=0, - elapsed_s=time.monotonic() - start, - dry_run=dry_run, - ) - - rows_total = v0_table.num_rows - rows_migrated = 0 - rows_skipped = 0 - - # Collect IDs already at v1 for idempotency check. - v1_existing = result_db.get_all_records(v1_path) - existing_ids: set[bytes] = set() - if v1_existing is not None and "__record_id" in v1_existing.schema.names: - existing_ids = {bytes(r) for r in v1_existing.column("__record_id").to_pylist() if r is not None} - - if progress: - logger.info("migrate_result_v0_to_v1: found %d rows at v0 path %r", rows_total, result_path) - - # Process in batches. - for batch_start in range(0, rows_total, batch_size): - batch = v0_table.slice(batch_start, batch_size) - - # Filter rows already at v1. - if existing_ids and "__record_id" in batch.schema.names: - mask = pa.array( - [bytes(rid) not in existing_ids for rid in batch.column("__record_id").to_pylist()], - type=pa.bool_(), - ) - new_rows = batch.filter(mask) - rows_skipped += batch.num_rows - new_rows.num_rows - else: - new_rows = batch - - if new_rows.num_rows == 0: - continue - - # Convert hash columns from string → binary. - transformed = _convert_hash_cols(new_rows) - - if not dry_run: - result_db.add_records(v1_path, transformed, skip_duplicates=True) - - rows_migrated += new_rows.num_rows - if progress: - logger.info( - "migrate_result_v0_to_v1: %d/%d rows processed", - min(batch_start + batch_size, rows_total), - rows_total, - ) - - if not dry_run: - result_db.flush() - - elapsed = time.monotonic() - start - return MigrationResult( - rows_total=rows_total, - rows_migrated=rows_migrated, - rows_skipped=rows_skipped, - rows_unresolvable=0, # rdb migration has no unresolvable rows - elapsed_s=elapsed, - dry_run=dry_run, - ) - - -def _convert_hash_cols(table: pa.Table) -> pa.Table: - """Convert all v0 large_string ContentHash columns to v1 large_binary. - - Args: - table: Arrow table with v0-format hash columns. - - Returns: - New table with hash columns converted to ``large_binary``. - """ - for col_name in _HASH_COLS: - if col_name not in table.schema.names: - continue - col_idx = table.schema.names.index(col_name) - string_vals = table.column(col_name).to_pylist() - binary_vals = pa.array( - [ - ContentHash.from_string(s).to_prefixed_digest() if s is not None else None - for s in string_vals - ], - type=pa.large_binary(), - ) - table = table.set_column(col_idx, col_name, binary_vals) - return table -``` - -- [ ] **Step 5: Create `src/orcapod/migrations/__init__.py`** (stub — full public API added in Task 9) - -```python -"""Orcapod schema migration utilities.""" -from orcapod.migrations.types import MigrationResult -from orcapod.migrations.result_db import migrate_result_v0_to_v1 - -__all__ = [ - "MigrationResult", - "migrate_result_v0_to_v1", -] -``` - -- [ ] **Step 6: Run the tests** - -```bash -uv run pytest tests/test_migrations/test_migration_types.py tests/test_migrations/test_result_db.py -v -``` -Expected: all PASSED - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/migrations/ tests/test_migrations/ -git commit -m "feat(migrations): add MigrationResult dataclass and migrate_result_v0_to_v1()" -``` - ---- - -## Task 9: Migration package — `pipeline_db.py` + `migrate_node()` + update `__init__.py` - -**Files:** -- Create: `src/orcapod/migrations/pipeline_db.py` -- Modify: `src/orcapod/migrations/__init__.py` -- Create: `tests/test_migrations/test_pipeline_db.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_migrations/test_pipeline_db.py`: - -```python -"""Tests for migrate_pipeline_v0_to_v1().""" -from __future__ import annotations - -import uuid -import pyarrow as pa -import pytest - -from orcapod.databases import InMemoryArrowDatabase -from orcapod.migrations.pipeline_db import migrate_pipeline_v0_to_v1 -from orcapod.migrations.types import MigrationResult -from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION, RESULT_DB_SCHEMA_VERSION, constants -from orcapod.types import ContentHash - - -_NODE_HASH = ContentHash("sha256", bytes(range(32))) -_INPUT_HASH = ContentHash("sha256", bytes(range(1, 33))) -_OUTPUT_HASH = ContentHash("sha256", bytes(range(2, 34))) - - -def _make_data_id() -> bytes: - return uuid.uuid7().bytes - - -def _write_v0_rdb_row(db: InMemoryArrowDatabase, rdb_path: tuple, data_id: bytes, input_hash: ContentHash) -> None: - """Write a minimal v0 rdb row that the pdb migration can look up.""" - row = pa.table({ - "__input_data_hash": pa.array([input_hash.to_string()], type=pa.large_string()), - "__pf_var_function_name": pa.array(["fn"], type=pa.large_string()), - "__pf_var_function_signature_hash": pa.array(["sha256:aabb"], type=pa.large_string()), - "__pf_var_function_content_hash": pa.array(["sha256:ccdd"], type=pa.large_string()), - "__pf_var_git_hash": pa.array(["abc"], type=pa.large_string()), - "__pf_exec_executor_type": pa.array(["local"], type=pa.large_string()), - "__pf_exec_python_version": pa.array(["3.11"], type=pa.large_string()), - "__pod_ts": pa.array([0], type=pa.timestamp("us", tz="UTC")), - "result": pa.array([99], type=pa.int64()), - }) - db.add_record(rdb_path, data_id, row) - db.flush() - - -def _write_v0_pdb_row(db: InMemoryArrowDatabase, pdb_path: tuple, data_id: bytes) -> None: - """Write a minimal v0 pdb row.""" - row = pa.table({ - constants.DATA_RECORD_ID: pa.array([data_id], type=pa.large_binary()), - constants.NODE_CONTENT_HASH_COL: pa.array([_NODE_HASH.to_string()], type=pa.large_string()), - constants.INPUT_DATA_HASH_COL: pa.array([_INPUT_HASH.to_string()], type=pa.large_string()), - constants.OUTPUT_DATA_HASH_COL: pa.array([_OUTPUT_HASH.to_string()], type=pa.large_string()), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array(["ctx"], type=pa.large_string()), - f"{constants.META_PREFIX}computed": pa.array([True], type=pa.bool_()), - constants.IS_EPHEMERAL_COL: pa.array([False], type=pa.bool_()), - "__pipeline_base_entry_id": pa.array([b"\x00" * 16], type=pa.large_binary()), - "__pipeline_recomputation_index": pa.array([0], type=pa.int32()), - }) - record_id = b"\x01" * 32 - db.add_record(pdb_path, record_id, row) - db.flush() - - -class TestMigratePipelineV0ToV1: - def test_happy_path_full_backfill(self): - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - data_id = _make_data_id() - - _write_v0_rdb_row(db, rdb_path, data_id, _INPUT_HASH) - _write_v0_pdb_row(db, pdb_path, data_id) - - result = migrate_pipeline_v0_to_v1( - db, pdb_path, db, rdb_path, progress=False - ) - - assert result.rows_total == 1 - assert result.rows_migrated == 1 - assert result.rows_unresolvable == 0 - assert result.rows_skipped == 0 - - def test_v1_hash_columns_are_binary(self): - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - data_id = _make_data_id() - - _write_v0_rdb_row(db, rdb_path, data_id, _INPUT_HASH) - _write_v0_pdb_row(db, pdb_path, data_id) - - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - assert v1_table is not None - assert v1_table.schema.field(constants.NODE_CONTENT_HASH_COL).type == pa.large_binary() - assert v1_table.schema.field(constants.INPUT_DATA_HASH_COL).type == pa.large_binary() - assert v1_table.schema.field(constants.OUTPUT_DATA_HASH_COL).type == pa.large_binary() - - def test_unresolvable_row_written_with_null_output_hash(self): - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - data_id = _make_data_id() - - # Write a pdb row whose data_id does NOT exist in rdb (ephemeral expired) - _write_v0_pdb_row(db, pdb_path, data_id) - # Do NOT write rdb row - - result = migrate_pipeline_v0_to_v1( - db, pdb_path, db, rdb_path, progress=False - ) - - assert result.rows_unresolvable == 1 - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - assert v1_table is not None - output_hash_col = v1_table.column(constants.OUTPUT_DATA_HASH_COL) - assert output_hash_col.to_pylist()[0] is None - - def test_idempotent(self): - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - data_id = _make_data_id() - - _write_v0_rdb_row(db, rdb_path, data_id, _INPUT_HASH) - _write_v0_pdb_row(db, pdb_path, data_id) - - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - result2 = migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - - assert result2.rows_migrated == 0 - assert result2.rows_skipped == 1 - - def test_dry_run_writes_nothing(self): - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - data_id = _make_data_id() - - _write_v0_rdb_row(db, rdb_path, data_id, _INPUT_HASH) - _write_v0_pdb_row(db, pdb_path, data_id) - - result = migrate_pipeline_v0_to_v1( - db, pdb_path, db, rdb_path, dry_run=True, progress=False - ) - - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - assert not db.table_exists(v1_path) - assert result.dry_run is True - assert result.rows_total == 1 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_migrations/test_pipeline_db.py -v -``` -Expected: `ImportError: cannot import name 'migrate_pipeline_v0_to_v1' from 'orcapod.migrations.pipeline_db'` - -- [ ] **Step 3: Create `src/orcapod/migrations/pipeline_db.py`** - -```python -"""v0 → v1 migration for the Orcapod pipeline DB (pdb).""" -from __future__ import annotations - -import logging -import time -from typing import TYPE_CHECKING - -import pyarrow as pa - -from orcapod.migrations.types import MigrationResult -from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION, RESULT_DB_SCHEMA_VERSION, constants -from orcapod.types import ContentHash - -if TYPE_CHECKING: - from orcapod.protocols.database_protocols import ArrowDatabaseProtocol - from orcapod.core.nodes.function_node import FunctionJobNode - -logger = logging.getLogger(__name__) - - -def migrate_pipeline_v0_to_v1( - pipeline_db: "ArrowDatabaseProtocol", - pipeline_path: tuple[str, ...], - result_db: "ArrowDatabaseProtocol", - result_path: tuple[str, ...], - *, - dry_run: bool = False, - batch_size: int = 500, - progress: bool = True, -) -> MigrationResult: - """Migrate a pipeline DB table from v0 schema to v1 schema. - - Reads records from ``pipeline_path`` (v0, no suffix), converts - ``__node_content_hash`` from ``large_string`` to ``large_binary``, - recovers ``__input_data_hash`` and ``__output_data_hash`` from the v0 - result DB, and writes the transformed rows to - ``pipeline_path + ("pdb_v1",)``. - - Rows whose result data cannot be found (e.g. ephemeral results that have - since expired) are written with a ``null`` ``__output_data_hash`` and - counted as ``rows_unresolvable``. - - Rows already present at the v1 path are skipped (idempotent re-runs). - - Args: - pipeline_db: The database containing the v0 pipeline table. - pipeline_path: Bare v0 pipeline path tuple (no ``pdb_v1`` suffix). - result_db: The database containing the v0 result table. - result_path: Bare v0 result path tuple (no ``rdb_v1`` suffix). - dry_run: If ``True``, read and count rows but write nothing. - batch_size: Number of rows to process per batch. - progress: If ``True``, log progress at INFO level. - - Returns: - ``MigrationResult`` summarising the run. - """ - v1_path = pipeline_path + (PIPELINE_DB_SCHEMA_VERSION,) - start = time.monotonic() - - v0_table = pipeline_db.get_all_records(pipeline_path) - if v0_table is None or v0_table.num_rows == 0: - logger.info("No v0 records found at %r — nothing to migrate.", pipeline_path) - return MigrationResult( - rows_total=0, - rows_migrated=0, - rows_skipped=0, - rows_unresolvable=0, - elapsed_s=time.monotonic() - start, - dry_run=dry_run, - ) - - rows_total = v0_table.num_rows - rows_migrated = 0 - rows_skipped = 0 - rows_unresolvable = 0 - - # Collect IDs already at v1 for idempotency. - v1_existing = pipeline_db.get_all_records(v1_path) - existing_ids: set[bytes] = set() - if v1_existing is not None and "__record_id" in v1_existing.schema.names: - existing_ids = { - bytes(r) - for r in v1_existing.column("__record_id").to_pylist() - if r is not None - } - - # Load the entire v0 result DB into memory for lookups by __data_id. - rdb_v0 = result_db.get_all_records(result_path) - rdb_index: dict[bytes, dict] = {} - if rdb_v0 is not None: - for row in rdb_v0.to_pylist(): - rid = bytes(row.get("__record_id") or b"") - if rid: - rdb_index[rid] = row - - if progress: - logger.info( - "migrate_pipeline_v0_to_v1: found %d rows at v0 path %r", - rows_total, - pipeline_path, - ) - - for batch_start in range(0, rows_total, batch_size): - batch = v0_table.slice(batch_start, batch_size) - - # Skip rows already at v1. - if existing_ids and "__record_id" in batch.schema.names: - mask = pa.array( - [ - bytes(rid) not in existing_ids - for rid in batch.column("__record_id").to_pylist() - ], - type=pa.bool_(), - ) - new_rows = batch.filter(mask) - rows_skipped += batch.num_rows - new_rows.num_rows - else: - new_rows = batch - - if new_rows.num_rows == 0: - continue - - transformed, batch_unresolvable = _transform_pdb_batch(new_rows, rdb_index) - rows_unresolvable += batch_unresolvable - - if not dry_run: - pipeline_db.add_records(v1_path, transformed, skip_duplicates=True) - - rows_migrated += new_rows.num_rows - if progress: - logger.info( - "migrate_pipeline_v0_to_v1: %d/%d rows processed", - min(batch_start + batch_size, rows_total), - rows_total, - ) - - if not dry_run: - pipeline_db.flush() - - elapsed = time.monotonic() - start - return MigrationResult( - rows_total=rows_total, - rows_migrated=rows_migrated, - rows_skipped=rows_skipped, - rows_unresolvable=rows_unresolvable, - elapsed_s=elapsed, - dry_run=dry_run, - ) - - -def _transform_pdb_batch( - batch: pa.Table, - rdb_index: dict[bytes, dict], -) -> tuple[pa.Table, int]: - """Transform a batch of v0 pdb rows into v1 format. - - Args: - batch: Arrow table slice of v0 pdb rows. - rdb_index: Dict mapping rdb record ID bytes to row dicts (from v0 rdb). - - Returns: - Tuple of (transformed Arrow table, count of unresolvable rows). - """ - node_hash_col = constants.NODE_CONTENT_HASH_COL - input_hash_col = constants.INPUT_DATA_HASH_COL - output_hash_col = constants.OUTPUT_DATA_HASH_COL - data_id_col = constants.DATA_RECORD_ID - - rows = batch.to_pylist() - unresolvable = 0 - out_rows: list[dict] = [] - - for row in rows: - new_row = dict(row) - - # Convert __node_content_hash from string → binary. - if node_hash_col in new_row and new_row[node_hash_col] is not None: - new_row[node_hash_col] = ContentHash.from_string( - new_row[node_hash_col] - ).to_prefixed_digest() - - # Recover __input_data_hash from rdb v0 row. - data_id = bytes(row.get(data_id_col) or b"") - rdb_row = rdb_index.get(data_id) - if rdb_row is not None: - raw_input_hash = rdb_row.get(input_hash_col) - if raw_input_hash is not None: - new_row[input_hash_col] = ContentHash.from_string( - raw_input_hash - ).to_prefixed_digest() - else: - new_row[input_hash_col] = None - # Recover __output_data_hash: re-encode from pdb v0 string column. - raw_output_hash = row.get(output_hash_col) - if raw_output_hash is not None: - new_row[output_hash_col] = ContentHash.from_string( - raw_output_hash - ).to_prefixed_digest() - else: - new_row[output_hash_col] = None - else: - # Result data gone (ephemeral expired or deleted). - new_row[input_hash_col] = None - new_row[output_hash_col] = None - unresolvable += 1 - - out_rows.append(new_row) - - # Rebuild Arrow table with corrected column types. - transformed = pa.Table.from_pylist(out_rows, schema=_v1_pdb_schema(batch)) - return transformed, unresolvable - - -def _v1_pdb_schema(v0_batch: pa.Table) -> pa.Schema: - """Derive the v1 pdb Arrow schema from a v0 batch. - - Replaces the three hash columns with ``large_binary`` equivalents; - all other columns retain their original types. - - Args: - v0_batch: A v0 pdb Arrow table (used to read non-hash column types). - - Returns: - Arrow schema for the v1 pdb table. - """ - binary_cols = { - constants.NODE_CONTENT_HASH_COL, - constants.INPUT_DATA_HASH_COL, - constants.OUTPUT_DATA_HASH_COL, - } - fields = [] - for field in v0_batch.schema: - if field.name in binary_cols: - fields.append(pa.field(field.name, pa.large_binary(), nullable=True)) - else: - fields.append(field) - # Ensure the new hash columns exist even if absent in v0. - existing_names = {f.name for f in fields} - for col in binary_cols: - if col not in existing_names: - fields.append(pa.field(col, pa.large_binary(), nullable=True)) - return pa.schema(fields) - - -def migrate_node( - node: "FunctionJobNode", - *, - dry_run: bool = False, - batch_size: int = 500, - progress: bool = True, -) -> MigrationResult: - """Convenience wrapper: migrate a single ``FunctionJobNode``'s pipeline DB. - - Extracts the pipeline DB, pipeline path, result DB, and result path - directly from the node and delegates to ``migrate_pipeline_v0_to_v1()``. - - Args: - node: The ``FunctionJobNode`` whose pipeline DB to migrate. - dry_run: If ``True``, read and count rows but write nothing. - batch_size: Rows to process per batch. - progress: If ``True``, log progress at INFO level. - - Returns: - ``MigrationResult`` summarising the run. - - Raises: - RuntimeError: If the node has no pipeline database attached. - """ - if node._pipeline_database is None: - raise RuntimeError( - f"Node {node.label!r} has no pipeline database — cannot migrate." - ) - cached_pod = node._cached_function_pod - if cached_pod is None: - raise RuntimeError( - f"Node {node.label!r} has no cached function pod — cannot locate result DB." - ) - return migrate_pipeline_v0_to_v1( - pipeline_db=node._pipeline_database, - pipeline_path=node.node_identity_path, - result_db=cached_pod._cache.result_database, - result_path=cached_pod._cache.record_path, - dry_run=dry_run, - batch_size=batch_size, - progress=progress, - ) -``` - -- [ ] **Step 4: Update `src/orcapod/migrations/__init__.py`** - -```python -"""Orcapod schema migration utilities. - -Public API: - ``migrate_pipeline_v0_to_v1`` — migrate a pipeline DB table from v0 to v1. - ``migrate_result_v0_to_v1`` — migrate a result DB table from v0 to v1. - ``migrate_node`` — convenience wrapper: migrate a ``FunctionJobNode`` in one call. - ``MigrationResult`` — dataclass summarising a migration run. -""" -from orcapod.migrations.types import MigrationResult -from orcapod.migrations.result_db import migrate_result_v0_to_v1 -from orcapod.migrations.pipeline_db import migrate_pipeline_v0_to_v1, migrate_node - -__all__ = [ - "MigrationResult", - "migrate_pipeline_v0_to_v1", - "migrate_result_v0_to_v1", - "migrate_node", -] -``` - -- [ ] **Step 5: Run the tests** - -```bash -uv run pytest tests/test_migrations/ -v --tb=short -``` -Expected: all PASSED - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/migrations/ tests/test_migrations/ -git commit -m "feat(migrations): add migrate_pipeline_v0_to_v1() and migrate_node() convenience wrapper" -``` - ---- - -## Task 10: CLI — `orcapod migrate` sub-commands - -**Files:** -- Create: `src/orcapod/cli/migrate.py` -- Modify: `src/orcapod/cli/__init__.py` -- Create: `tests/test_cli/test_migrate.py` - -- [ ] **Step 1: Write failing CLI smoke test** - -Create `tests/test_cli/test_migrate.py`: - -```python -"""Smoke tests for the `orcapod migrate` CLI sub-commands.""" -from __future__ import annotations - -import json -import subprocess -import sys - - -def _run(*args: str) -> subprocess.CompletedProcess: - return subprocess.run( - [sys.executable, "-m", "orcapod.cli", *args], - capture_output=True, - text=True, - ) - - -class TestMigratePipelineDbCli: - def test_help_exits_zero(self): - result = subprocess.run( - ["uv", "run", "orcapod", "migrate", "pipeline-db", "--help"], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert "PIPELINE_DB_PATH" in result.stdout or "pipeline" in result.stdout.lower() - - def test_dry_run_exits_zero(self, tmp_path): - """--dry-run with a non-existent DB path exits 0 (nothing to migrate).""" - db_path = str(tmp_path / "pipeline_db") - result = subprocess.run( - [ - "uv", "run", "orcapod", "migrate", "pipeline-db", - db_path, db_path, "my_node/path", - "--dry-run", "--no-progress", - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - - def test_json_summary_output(self, tmp_path): - db_path = str(tmp_path / "pipeline_db") - result = subprocess.run( - [ - "uv", "run", "orcapod", "migrate", "pipeline-db", - db_path, db_path, "my_node/path", - "--dry-run", "--json-summary", "--no-progress", - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - summary = json.loads(result.stdout.strip()) - assert "rows_total" in summary - assert "dry_run" in summary - assert summary["dry_run"] is True - - -class TestMigrateResultDbCli: - def test_help_exits_zero(self): - result = subprocess.run( - ["uv", "run", "orcapod", "migrate", "result-db", "--help"], - capture_output=True, - text=True, - ) - assert result.returncode == 0 - - def test_dry_run_exits_zero(self, tmp_path): - db_path = str(tmp_path / "result_db") - result = subprocess.run( - [ - "uv", "run", "orcapod", "migrate", "result-db", - db_path, "my_pod/path", - "--dry-run", "--no-progress", - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_cli/test_migrate.py -v -``` -Expected: `SystemExit` or `AssertionError` because `orcapod migrate` doesn't exist yet. - -- [ ] **Step 3: Create `src/orcapod/cli/migrate.py`** - -```python -"""``orcapod migrate`` sub-commands. - -Provides ``orcapod migrate pipeline-db`` and ``orcapod migrate result-db`` -for upgrading v0 pipeline/result DB tables to the v1 schema. -""" -from __future__ import annotations - -import json -import sys -from pathlib import Path - -import typer - -from orcapod.databases.delta_lake_databases import DeltaTableDatabase - -migrate_app = typer.Typer( - name="migrate", - help="Migrate Orcapod pipeline and result DB tables to the current schema version.", - no_args_is_help=True, -) - - -@migrate_app.command("pipeline-db") -def migrate_pipeline_db( - pipeline_db_path: str = typer.Argument(..., help="Path to the pipeline DB (Delta Lake root)."), - result_db_path: str = typer.Argument(..., help="Path to the result DB (Delta Lake root)."), - node_paths: list[str] = typer.Argument(..., help="One or more bare v0 node paths (e.g. 'my_node/schema:abc123')."), - dry_run: bool = typer.Option(False, "--dry-run", help="Count rows to migrate without writing."), - batch_size: int = typer.Option(500, "--batch-size", help="Rows processed per batch."), - progress: bool = typer.Option(True, "--progress/--no-progress", help="Log progress messages."), - json_summary: bool = typer.Option(False, "--json-summary", help="Print JSON summary to stdout on completion."), -) -> None: - """Migrate one or more pipeline DB node paths from v0 to v1 schema.""" - from orcapod.migrations.pipeline_db import migrate_pipeline_v0_to_v1 - - pipeline_db = DeltaTableDatabase(base_path=pipeline_db_path) - result_db = DeltaTableDatabase(base_path=result_db_path) - - for node_path_str in node_paths: - pipeline_path = tuple(node_path_str.split("/")) - result_path = pipeline_path # by convention result DB mirrors pipeline path - - if progress: - typer.echo(f"Migrating pipeline DB: {pipeline_db_path}") - typer.echo(f" node path: {node_path_str}") - - result = migrate_pipeline_v0_to_v1( - pipeline_db=pipeline_db, - pipeline_path=pipeline_path, - result_db=result_db, - result_path=result_path, - dry_run=dry_run, - batch_size=batch_size, - progress=progress, - ) - - if progress: - typer.echo( - f" migrated: {result.rows_migrated} " - f"skipped (already v1): {result.rows_skipped} " - f"unresolvable: {result.rows_unresolvable}" - ) - typer.echo(f" elapsed: {result.elapsed_s:.1f}s") - - if json_summary: - summary = { - "rows_total": result.rows_total, - "rows_migrated": result.rows_migrated, - "rows_skipped": result.rows_skipped, - "rows_unresolvable": result.rows_unresolvable, - "elapsed_s": result.elapsed_s, - "dry_run": result.dry_run, - } - typer.echo(json.dumps(summary)) - - -@migrate_app.command("result-db") -def migrate_result_db( - result_db_path: str = typer.Argument(..., help="Path to the result DB (Delta Lake root)."), - record_paths: list[str] = typer.Argument(..., help="One or more bare v0 record paths."), - dry_run: bool = typer.Option(False, "--dry-run", help="Count rows to migrate without writing."), - batch_size: int = typer.Option(500, "--batch-size", help="Rows processed per batch."), - progress: bool = typer.Option(True, "--progress/--no-progress", help="Log progress messages."), - json_summary: bool = typer.Option(False, "--json-summary", help="Print JSON summary to stdout on completion."), -) -> None: - """Migrate one or more result DB record paths from v0 to v1 schema.""" - from orcapod.migrations.result_db import migrate_result_v0_to_v1 - - result_db = DeltaTableDatabase(base_path=result_db_path) - - for record_path_str in record_paths: - result_path = tuple(record_path_str.split("/")) - - if progress: - typer.echo(f"Migrating result DB: {result_db_path}") - typer.echo(f" record path: {record_path_str}") - - result = migrate_result_v0_to_v1( - result_db=result_db, - result_path=result_path, - dry_run=dry_run, - batch_size=batch_size, - progress=progress, - ) - - if progress: - typer.echo( - f" migrated: {result.rows_migrated} " - f"skipped (already v1): {result.rows_skipped}" - ) - typer.echo(f" elapsed: {result.elapsed_s:.1f}s") - - if json_summary: - summary = { - "rows_total": result.rows_total, - "rows_migrated": result.rows_migrated, - "rows_skipped": result.rows_skipped, - "rows_unresolvable": result.rows_unresolvable, - "elapsed_s": result.elapsed_s, - "dry_run": result.dry_run, - } - typer.echo(json.dumps(summary)) -``` - -- [ ] **Step 4: Register `migrate_app` in `src/orcapod/cli/__init__.py`** - -Add to `src/orcapod/cli/__init__.py`: - -```python -from orcapod.cli.migrate import migrate_app - -app.add_typer(migrate_app, name="migrate") -``` - -- [ ] **Step 5: Run the CLI tests** - -```bash -uv run pytest tests/test_cli/test_migrate.py -v --tb=short -``` -Expected: all PASSED - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/cli/migrate.py src/orcapod/cli/__init__.py tests/test_cli/test_migrate.py -git commit -m "feat(cli): add orcapod migrate pipeline-db and result-db sub-commands" -``` - ---- - -## Task 11: Create committed test fixtures (sample Delta Lake tables) - -**Files:** -- Create: `tests/fixtures/` directory with four sample Delta Lake tables -- Create: `tests/fixtures/generate_fixtures.py` — script to regenerate fixtures - -The fixtures represent a pair of pdb tables (v0 and v1) and a pair of rdb tables (v0 and v1) containing the same three logical rows. They are committed to git and serve as golden references for migration tests. - -- [ ] **Step 1: Create the fixture-generation script** - -Create `tests/fixtures/generate_fixtures.py`: - -```python -"""Script to generate sample fixture Delta Lake tables for migration tests. - -Run once and commit the output: - uv run python tests/fixtures/generate_fixtures.py - -The generated tables are stored under tests/fixtures/ and checked into git. -They are the authoritative golden references for migration correctness tests. -""" -from __future__ import annotations - -import sys -from pathlib import Path - -# Add project src to path -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) - -import pyarrow as pa -from orcapod.databases.delta_lake_databases import DeltaTableDatabase -from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION, RESULT_DB_SCHEMA_VERSION, constants -from orcapod.types import ContentHash - -FIXTURES_DIR = Path(__file__).parent - -# --- Shared test data --- -_DATA_ID_1 = b"\x01" * 16 -_DATA_ID_2 = b"\x02" * 16 -_DATA_ID_3 = b"\x03" * 16 - -_NODE_HASH = ContentHash("sha256", b"\xaa" * 32) -_INPUT_HASH_1 = ContentHash("sha256", b"\x11" * 32) -_INPUT_HASH_2 = ContentHash("sha256", b"\x22" * 32) -_INPUT_HASH_3 = ContentHash("sha256", b"\x33" * 32) -_OUTPUT_HASH_1 = ContentHash("sha256", b"\x44" * 32) -_OUTPUT_HASH_2 = ContentHash("sha256", b"\x55" * 32) -_OUTPUT_HASH_3 = ContentHash("sha256", b"\x66" * 32) -_SIG_HASH = ContentHash("sha256", b"\x77" * 32) -_CONTENT_HASH = ContentHash("sha256", b"\x88" * 32) - - -def write_pdb_v0(): - db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "pdb_v0_sample")) - path = ("node",) - rows = pa.table({ - constants.DATA_RECORD_ID: pa.array([_DATA_ID_1, _DATA_ID_2, _DATA_ID_3], type=pa.large_binary()), - constants.NODE_CONTENT_HASH_COL: pa.array( - [_NODE_HASH.to_string()] * 3, type=pa.large_string() - ), - constants.INPUT_DATA_HASH_COL: pa.array( - [h.to_string() for h in [_INPUT_HASH_1, _INPUT_HASH_2, _INPUT_HASH_3]], - type=pa.large_string(), - ), - constants.OUTPUT_DATA_HASH_COL: pa.array( - [h.to_string() for h in [_OUTPUT_HASH_1, _OUTPUT_HASH_2, _OUTPUT_HASH_3]], - type=pa.large_string(), - ), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array( - ["ctx1", "ctx2", "ctx3"], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array([True, True, False], type=pa.bool_()), - constants.IS_EPHEMERAL_COL: pa.array([False, False, False], type=pa.bool_()), - "__pipeline_base_entry_id": pa.array( - [_DATA_ID_1, _DATA_ID_2, _DATA_ID_3], type=pa.large_binary() - ), - "__pipeline_recomputation_index": pa.array([0, 0, 0], type=pa.int32()), - }) - record_ids = [b"\xr1" * 32 for b in [b"\x01", b"\x02", b"\x03"]] - # Use simple unique IDs - db.add_records(path, rows, skip_duplicates=False) - db.flush() - print(f"Written pdb_v0_sample: {rows.num_rows} rows") - - -def write_pdb_v1(): - db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "pdb_v1_sample")) - path = ("node", PIPELINE_DB_SCHEMA_VERSION) - rows = pa.table({ - constants.DATA_RECORD_ID: pa.array([_DATA_ID_1, _DATA_ID_2, _DATA_ID_3], type=pa.large_binary()), - constants.NODE_CONTENT_HASH_COL: pa.array( - [_NODE_HASH.to_prefixed_digest()] * 3, type=pa.large_binary() - ), - constants.INPUT_DATA_HASH_COL: pa.array( - [h.to_prefixed_digest() for h in [_INPUT_HASH_1, _INPUT_HASH_2, _INPUT_HASH_3]], - type=pa.large_binary(), - ), - constants.OUTPUT_DATA_HASH_COL: pa.array( - [h.to_prefixed_digest() for h in [_OUTPUT_HASH_1, _OUTPUT_HASH_2, _OUTPUT_HASH_3]], - type=pa.large_binary(), - ), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array( - ["ctx1", "ctx2", "ctx3"], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array([True, True, False], type=pa.bool_()), - constants.IS_EPHEMERAL_COL: pa.array([False, False, False], type=pa.bool_()), - "__pipeline_base_entry_id": pa.array( - [_DATA_ID_1, _DATA_ID_2, _DATA_ID_3], type=pa.large_binary() - ), - "__pipeline_recomputation_index": pa.array([0, 0, 0], type=pa.int32()), - }) - db.add_records(path, rows, skip_duplicates=False) - db.flush() - print(f"Written pdb_v1_sample: {rows.num_rows} rows") - - -def write_rdb_v0(): - db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "rdb_v0_sample")) - path = ("pod",) - rows = pa.table({ - "__input_data_hash": pa.array( - [h.to_string() for h in [_INPUT_HASH_1, _INPUT_HASH_2, _INPUT_HASH_3]], - type=pa.large_string(), - ), - "__pf_var_function_name": pa.array(["my_func"] * 3, type=pa.large_string()), - "__pf_var_function_signature_hash": pa.array( - [_SIG_HASH.to_string()] * 3, type=pa.large_string() - ), - "__pf_var_function_content_hash": pa.array( - [_CONTENT_HASH.to_string()] * 3, type=pa.large_string() - ), - "__pf_var_git_hash": pa.array(["abc123"] * 3, type=pa.large_string()), - "__pf_exec_executor_type": pa.array(["local"] * 3, type=pa.large_string()), - "__pf_exec_python_version": pa.array(["3.11"] * 3, type=pa.large_string()), - "__pod_ts": pa.array([0, 1, 2], type=pa.timestamp("us", tz="UTC")), - "result": pa.array([10, 20, 30], type=pa.int64()), - }) - for i, data_id in enumerate([_DATA_ID_1, _DATA_ID_2, _DATA_ID_3]): - db.add_record(path, data_id, rows.slice(i, 1)) - db.flush() - print(f"Written rdb_v0_sample: {rows.num_rows} rows") - - -def write_rdb_v1(): - db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "rdb_v1_sample")) - path = ("pod", RESULT_DB_SCHEMA_VERSION) - rows = pa.table({ - "__input_data_hash": pa.array( - [h.to_prefixed_digest() for h in [_INPUT_HASH_1, _INPUT_HASH_2, _INPUT_HASH_3]], - type=pa.large_binary(), - ), - "__pf_var_function_name": pa.array(["my_func"] * 3, type=pa.large_string()), - "__pf_var_function_signature_hash": pa.array( - [_SIG_HASH.to_prefixed_digest()] * 3, type=pa.large_binary() - ), - "__pf_var_function_content_hash": pa.array( - [_CONTENT_HASH.to_prefixed_digest()] * 3, type=pa.large_binary() - ), - "__pf_var_git_hash": pa.array(["abc123"] * 3, type=pa.large_string()), - "__pf_exec_executor_type": pa.array(["local"] * 3, type=pa.large_string()), - "__pf_exec_python_version": pa.array(["3.11"] * 3, type=pa.large_string()), - "__pod_ts": pa.array([0, 1, 2], type=pa.timestamp("us", tz="UTC")), - "result": pa.array([10, 20, 30], type=pa.int64()), - }) - for i, data_id in enumerate([_DATA_ID_1, _DATA_ID_2, _DATA_ID_3]): - db.add_record(path, data_id, rows.slice(i, 1)) - db.flush() - print(f"Written rdb_v1_sample: {rows.num_rows} rows") - - -if __name__ == "__main__": - write_pdb_v0() - write_pdb_v1() - write_rdb_v0() - write_rdb_v1() - print("All fixtures written.") -``` - -- [ ] **Step 2: Run the generation script** - -```bash -uv run python tests/fixtures/generate_fixtures.py -``` -Expected output: -``` -Written pdb_v0_sample: 3 rows -Written pdb_v1_sample: 3 rows -Written rdb_v0_sample: 3 rows -Written rdb_v1_sample: 3 rows -All fixtures written. -``` - -- [ ] **Step 3: Verify fixtures were created** - -```bash -ls tests/fixtures/ -``` -Expected: `generate_fixtures.py pdb_v0_sample/ pdb_v1_sample/ rdb_v0_sample/ rdb_v1_sample/` - -- [ ] **Step 4: Commit the fixtures** - -```bash -git add tests/fixtures/ -git commit -m "test(fixtures): add committed sample Delta Lake tables for pdb/rdb v0 and v1 schema versions" -``` - ---- - -## Task 12: Golden migration tests - -**Files:** -- Create: `tests/test_migrations/test_golden.py` - -These tests are the authoritative correctness check: migrate the v0 fixture and assert the output matches the v1 fixture row-for-row. - -- [ ] **Step 1: Write the golden tests** - -Create `tests/test_migrations/test_golden.py`: - -```python -"""Golden migration tests. - -Migrates the committed v0 fixture tables and asserts the output -matches the committed v1 fixtures row-for-row. - -Maintenance rule: when a new schema version vX is introduced, add -new v{X-1} and vX fixtures (do not modify existing ones), then add a -new test here following the same pattern. -""" -from __future__ import annotations - -from pathlib import Path - -import pyarrow as pa -import pytest - -from orcapod.databases.delta_lake_databases import DeltaTableDatabase -from orcapod.databases import InMemoryArrowDatabase -from orcapod.migrations.pipeline_db import migrate_pipeline_v0_to_v1 -from orcapod.migrations.result_db import migrate_result_v0_to_v1 -from orcapod.system_constants import PIPELINE_DB_SCHEMA_VERSION, RESULT_DB_SCHEMA_VERSION - -FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" - - -def _sort_table(table: pa.Table) -> pa.Table: - """Sort by all columns for stable comparison.""" - # Sort by the first column (record_id or input_data_hash) for determinism. - if table.num_rows == 0: - return table - sort_col = table.schema.names[0] - indices = pa.compute.sort_indices(table, sort_keys=[(sort_col, "ascending")]) - return table.take(indices) - - -def _tables_equal(a: pa.Table, b: pa.Table) -> bool: - """Return True if two tables contain the same data (same schema and rows, order-independent).""" - if set(a.schema.names) != set(b.schema.names): - return False - # Align column order. - b_aligned = b.select(a.schema.names) - a_sorted = _sort_table(a) - b_sorted = _sort_table(b_aligned) - return a_sorted.equals(b_sorted) - - -class TestGoldenPdbMigration: - """Migrate pdb_v0_sample and compare against pdb_v1_sample.""" - - def test_migrate_pdb_v0_to_v1_matches_golden(self, tmp_path): - # Load v0 fixture from Delta Lake. - v0_db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "pdb_v0_sample")) - rdb_v0_db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "rdb_v0_sample")) - - # Write migration output to a temp in-memory DB. - out_db = InMemoryArrowDatabase() - - # Mirror the v0 data into out_db so migrate_pipeline_v0_to_v1 can read it. - # (The function reads from pipeline_db at pipeline_path.) - # We'll use a wrapper that reads from v0_db for v0 and writes to out_db for v1. - # Simpler: copy v0 data into out_db first, then migrate within out_db. - pdb_v0_path = ("node",) - rdb_v0_path = ("pod",) - - v0_table = v0_db.get_all_records(pdb_v0_path) - assert v0_table is not None, "pdb_v0_sample fixture missing" - out_db.add_records(pdb_v0_path, v0_table, skip_duplicates=False) - out_db.flush() - - rdb_v0_table = rdb_v0_db.get_all_records(rdb_v0_path) - assert rdb_v0_table is not None, "rdb_v0_sample fixture missing" - rdb_out_db = InMemoryArrowDatabase() - rdb_out_db.add_records(rdb_v0_path, rdb_v0_table, skip_duplicates=False) - rdb_out_db.flush() - - # Run migration. - result = migrate_pipeline_v0_to_v1( - pipeline_db=out_db, - pipeline_path=pdb_v0_path, - result_db=rdb_out_db, - result_path=rdb_v0_path, - progress=False, - ) - - assert result.rows_total == 3 - assert result.rows_migrated == 3 - assert result.rows_unresolvable == 0 - - # Read migrated output. - migrated = out_db.get_all_records(pdb_v0_path + (PIPELINE_DB_SCHEMA_VERSION,)) - assert migrated is not None - - # Read golden v1 fixture. - v1_db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "pdb_v1_sample")) - golden = v1_db.get_all_records(("node", PIPELINE_DB_SCHEMA_VERSION)) - assert golden is not None, "pdb_v1_sample fixture missing" - - # Compare. - assert _tables_equal( - migrated.select(golden.schema.names), - golden, - ), "Migrated pdb table does not match golden v1 fixture" - - -class TestGoldenRdbMigration: - """Migrate rdb_v0_sample and compare against rdb_v1_sample.""" - - def test_migrate_rdb_v0_to_v1_matches_golden(self): - v0_db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "rdb_v0_sample")) - - rdb_v0_path = ("pod",) - v0_table = v0_db.get_all_records(rdb_v0_path) - assert v0_table is not None, "rdb_v0_sample fixture missing" - - out_db = InMemoryArrowDatabase() - out_db.add_records(rdb_v0_path, v0_table, skip_duplicates=False) - out_db.flush() - - result = migrate_result_v0_to_v1( - result_db=out_db, - result_path=rdb_v0_path, - progress=False, - ) - - assert result.rows_total == 3 - assert result.rows_migrated == 3 - assert result.rows_unresolvable == 0 - - migrated = out_db.get_all_records(rdb_v0_path + (RESULT_DB_SCHEMA_VERSION,)) - assert migrated is not None - - v1_db = DeltaTableDatabase(base_path=str(FIXTURES_DIR / "rdb_v1_sample")) - golden = v1_db.get_all_records(("pod", RESULT_DB_SCHEMA_VERSION)) - assert golden is not None, "rdb_v1_sample fixture missing" - - assert _tables_equal( - migrated.select(golden.schema.names), - golden, - ), "Migrated rdb table does not match golden v1 fixture" -``` - -- [ ] **Step 2: Run the golden tests** - -```bash -uv run pytest tests/test_migrations/test_golden.py -v --tb=short -``` -Expected: 2 PASSED - -- [ ] **Step 3: Run the full test suite to check for regressions** - -```bash -uv run pytest --tb=short -q -``` -Expected: all PASSED (no regressions) - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_migrations/test_golden.py -git commit -m "test(migrations): add golden migration tests against committed fixtures" -``` - ---- - -## Final: Full regression check + push - -- [ ] **Run the complete test suite** - -```bash -uv run pytest --tb=short -q -``` -Expected: all PASSED - -- [ ] **Push the feature branch** - -```bash -git push -u origin HEAD -``` diff --git a/superpowers/plans/2026-07-21-itl-533-remove-node-content-hash-preimage.md b/superpowers/plans/2026-07-21-itl-533-remove-node-content-hash-preimage.md deleted file mode 100644 index 1271a56db..000000000 --- a/superpowers/plans/2026-07-21-itl-533-remove-node-content-hash-preimage.md +++ /dev/null @@ -1,1199 +0,0 @@ -# ITL-533: Remove NODE_CONTENT_HASH_COL from record_id preimage — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove `NODE_CONTENT_HASH_COL` (`_node_content_hash`) from the `record_id` preimage, stored pdb rows, and the side-effect tdb path — making the pdb v1 schema authoritative without the redundant column. - -**Architecture:** Six targeted file changes, TDD throughout. Shared preimage helper extracted to `function_node.py` and imported by `side_effects.py`. `_filter_by_content_hash()` deleted entirely. pdb migration extended to drop the column and recompute hashes. tdb path gains formal versioning via `TRACKING_DB_SCHEMA_VERSION = "tdb_v1"`. - -**Tech Stack:** Python, PyArrow, `InMemoryArrowDatabase`, `uv run pytest` - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/system_constants.py` | Add `TRACKING_DB_SCHEMA_VERSION = "tdb_v1"` | -| `src/orcapod/core/nodes/function_node.py` | Extract `_build_record_id_preimage()`, update `_build_entry_id_preimage()`, remove `NODE_CONTENT_HASH_COL` from `add_pipeline_record()`, delete `_filter_by_content_hash()`, clean up `get_all_records()`, remove `pc` lazy import | -| `src/orcapod/side_effects.py` | Remove `node_content_hash_str` from `_execute_side_effect_row()` and 4 call sites, import `_build_record_id_preimage`, update `attach_databases()` to append `TRACKING_DB_SCHEMA_VERSION` | -| `src/orcapod/migrations/pipeline_db.py` | Remove `NODE_CONTENT_HASH_COL` from `_PDB_HASH_COLS`, update `_v1_pdb_schema()` to drop the column, extend `_transform_pdb_batch()` to recompute `__pipeline_base_entry_id` and `__record_id` | -| `tests/test_migrations/test_pipeline_db.py` | Update 2 existing assertions, add 6 new test cases | -| `tests/test_core/function_pod/test_node_content_hash_redundancy.py` | Already has the pre-written failing test — no edits needed here | - ---- - -### Task 1: Add `TRACKING_DB_SCHEMA_VERSION` to `system_constants.py` - -**Files:** -- Modify: `src/orcapod/system_constants.py` - -- [ ] **Step 1: Add the constant** - -In `src/orcapod/system_constants.py`, after line 24 (`RESULT_DB_SCHEMA_VERSION = "rdb_v1"`), add: - -```python -TRACKING_DB_SCHEMA_VERSION = "tdb_v1" -``` - -The module-level section (lines 23–25) becomes: - -```python -PIPELINE_DB_SCHEMA_VERSION = "pdb_v1" -RESULT_DB_SCHEMA_VERSION = "rdb_v1" -TRACKING_DB_SCHEMA_VERSION = "tdb_v1" -``` - -- [ ] **Step 2: Verify import works** - -```bash -cd /home/kurouto/kurouto-jobs/dbe06232-16a2-4ea3-a06c-8908d94b8161/orcapod-python -uv run python -c "from orcapod.system_constants import TRACKING_DB_SCHEMA_VERSION; print(TRACKING_DB_SCHEMA_VERSION)" -``` - -Expected output: `tdb_v1` - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/system_constants.py -git commit -m "feat(system_constants): add TRACKING_DB_SCHEMA_VERSION = 'tdb_v1'" -``` - ---- - -### Task 2: Extract `_build_record_id_preimage()` and update preimage methods - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` -- Test: `tests/test_core/function_pod/test_node_content_hash_redundancy.py` (already written — `test_node_content_hash_col_not_in_preimage_keys`) - -- [ ] **Step 1: Confirm the red test** - -```bash -uv run pytest tests/test_core/function_pod/test_node_content_hash_redundancy.py::TestPreimageShape::test_node_content_hash_col_not_in_preimage_keys -v -``` - -Expected: `FAILED` — `_node_content_hash` is currently in the preimage. - -- [ ] **Step 2: Add `_build_record_id_preimage()` module-level function** - -In `src/orcapod/core/nodes/function_node.py`, after line 81 (after `_PIPELINE_RECOMPUTATION_INDEX_COL = "__pipeline_recomputation_index"`), insert this new module-level private function: - -```python -def _build_record_id_preimage( - tag: "TagProtocol", - input_data: "DataProtocol", -) -> "pa.Table": - """Build the Arrow preimage table for record_id / base_entry_id computation. - - Combines the tag's system-tag columns with the binary input data hash into a - single-row Arrow table. ``NODE_CONTENT_HASH_COL`` is intentionally excluded — - it is redundant (fully determined by the table path + system tags). - - Args: - tag: The tag datagram for the input row. - input_data: The data datagram for the input row. - - Returns: - A single-row ``pa.Table`` with system-tag columns and - ``INPUT_DATA_HASH_COL`` (``large_binary``). - """ - return tag.as_table(columns={"system_tags": True}).append_column( - constants.INPUT_DATA_HASH_COL, - pa.array([input_data.content_hash().to_prefixed_digest()], type=pa.large_binary()), - ) -``` - -- [ ] **Step 3: Update `_build_entry_id_preimage()` to delegate** - -`_build_entry_id_preimage` is the method on `FunctionJobNode` (around line 1559). -Replace its entire body with a delegation call: - -Old: -```python - def _build_entry_id_preimage( - self, - tag: TagProtocol, - input_data: DataProtocol, - ) -> pa.Table: - """Builds the shared Arrow preimage used by both entry-ID methods. - - Combines the tag's system columns with the input data hash and - node content hash into a single-row Arrow table. - - Args: - tag: The tag datagram for the input row. - input_data: The data datagram for the input row. - - Returns: - A single-row ``pa.Table`` with system-tag columns, - ``INPUT_DATA_HASH_COL``, and ``NODE_CONTENT_HASH_COL``. - """ - return ( - tag.as_table(columns={"system_tags": True}) - .append_column( - constants.INPUT_DATA_HASH_COL, - pa.array([input_data.content_hash().to_string()], type=pa.large_string()), - ) - .append_column( - constants.NODE_CONTENT_HASH_COL, - pa.array([self.content_hash().to_string()], type=pa.large_string()), - ) - ) -``` - -New: -```python - def _build_entry_id_preimage( - self, - tag: TagProtocol, - input_data: DataProtocol, - ) -> pa.Table: - """Builds the shared Arrow preimage used by both entry-ID methods. - - Delegates to the module-level ``_build_record_id_preimage()`` helper. - The preimage contains system-tag columns and ``INPUT_DATA_HASH_COL`` - (``large_binary``). ``NODE_CONTENT_HASH_COL`` is excluded — it is - redundant and fully determined by the table path and system tags. - - Args: - tag: The tag datagram for the input row. - input_data: The data datagram for the input row. - - Returns: - A single-row ``pa.Table`` with system-tag columns and - ``INPUT_DATA_HASH_COL``. - """ - return _build_record_id_preimage(tag, input_data) -``` - -- [ ] **Step 4: Update `compute_pipeline_entry_id()` docstring** - -Find the docstring of `compute_pipeline_entry_id` (around line 1615). Remove the sentence that mentions `NODE_CONTENT_HASH_COL`. Replace: - -``` - ``NODE_CONTENT_HASH_COL`` is always included so that two runs processing - identical inputs each get a distinct entry ID, regardless of table scope. -``` - -With: - -``` - The preimage is ``system_tags + INPUT_DATA_HASH_COL + recomputation_index``. -``` - -- [ ] **Step 5: Update `compute_base_entry_id()` docstring** - -Find the docstring (around line 1589). Replace the sentence: - -``` - This value is identical to the pre-ITL-508 ``compute_pipeline_entry_id`` output: - it hashes the tag's system columns plus ``INPUT_DATA_HASH_COL`` and - ``NODE_CONTENT_HASH_COL``. -``` - -With: - -``` - Hashes the tag's system-tag columns plus ``INPUT_DATA_HASH_COL`` (binary). -``` - -- [ ] **Step 6: Run the previously-red test — it should now pass** - -```bash -uv run pytest tests/test_core/function_pod/test_node_content_hash_redundancy.py::TestPreimageShape::test_node_content_hash_col_not_in_preimage_keys -v -``` - -Expected: `PASSED` - -- [ ] **Step 7: Run the full redundancy test file to confirm no regressions** - -```bash -uv run pytest tests/test_core/function_pod/test_node_content_hash_redundancy.py -v -``` - -Expected: 11 pass (the previously-red test now green), 0 fail. - -- [ ] **Step 8: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "refactor(function_node): extract _build_record_id_preimage, drop NODE_CONTENT_HASH_COL from preimage" -``` - ---- - -### Task 3: Remove `NODE_CONTENT_HASH_COL` from `add_pipeline_record()` - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` -- Test: `tests/test_core/function_pod/test_node_content_hash_redundancy.py` (new test) - -- [ ] **Step 1: Write the failing test** - -Add this test to `TestPreimageShape` in `tests/test_core/function_pod/test_node_content_hash_redundancy.py`: - -```python - def test_add_pipeline_record_does_not_store_node_content_hash(self, double_pf): - """``add_pipeline_record()`` must not write ``NODE_CONTENT_HASH_COL`` to the DB. - - After ITL-533, the pdb_v1 schema excludes ``__node_content_hash``. - """ - import uuid - from orcapod.databases import InMemoryArrowDatabase - - db = InMemoryArrowDatabase() - src = _make_source_stream([42]) - node = FunctionJobNode( - function_pod=FunctionPod(data_function=double_pf), - input_stream=src, - pipeline_database=db, - ) - tag, data = next(iter(src.iter_data())) - node.add_pipeline_record( - tag=tag, - input_data=data, - data_record_id=uuid.uuid4(), - computed=True, - ) - table = db.get_all_records(node._versioned_pipeline_path) - assert table is not None, "Pipeline record was not written." - assert constants.NODE_CONTENT_HASH_COL not in table.column_names, ( - f"``{constants.NODE_CONTENT_HASH_COL}`` must not be stored in pdb_v1 rows." - ) -``` - -- [ ] **Step 2: Run to confirm it fails** - -```bash -uv run pytest "tests/test_core/function_pod/test_node_content_hash_redundancy.py::TestPreimageShape::test_add_pipeline_record_does_not_store_node_content_hash" -v -``` - -Expected: `FAILED` - -- [ ] **Step 3: Remove `NODE_CONTENT_HASH_COL` from `add_pipeline_record()`** - -In `src/orcapod/core/nodes/function_node.py`, find the `meta_table = pa.table({...})` block inside `add_pipeline_record()` (around line 1725). Remove these two lines: - -```python - constants.NODE_CONTENT_HASH_COL: pa.array( - [self.content_hash().to_prefixed_digest()], type=pa.large_binary() - ), -``` - -Also update the `add_pipeline_record()` docstring. Remove the bullet: -``` - - Output data record ID (for joining with result records) -``` - -... wait, that one stays. Remove only the implicit NODE_CONTENT_HASH_COL mention — the docstring's bullet list doesn't actually list it explicitly, so no docstring change is needed here. - -- [ ] **Step 4: Run the new test — it should pass** - -```bash -uv run pytest "tests/test_core/function_pod/test_node_content_hash_redundancy.py::TestPreimageShape::test_add_pipeline_record_does_not_store_node_content_hash" -v -``` - -Expected: `PASSED` - -- [ ] **Step 5: Run the full redundancy test file** - -```bash -uv run pytest tests/test_core/function_pod/test_node_content_hash_redundancy.py -v -``` - -Expected: 12 pass, 0 fail. - -- [ ] **Step 6: Commit** - -```bash -git add tests/test_core/function_pod/test_node_content_hash_redundancy.py \ - src/orcapod/core/nodes/function_node.py -git commit -m "feat(function_node): remove NODE_CONTENT_HASH_COL from add_pipeline_record output" -``` - ---- - -### Task 4: Delete `_filter_by_content_hash()` and clean up references - -**Files:** -- Modify: `src/orcapod/core/nodes/function_node.py` - -- [ ] **Step 1: Delete `_filter_by_content_hash()` method** - -Find and delete the entire method in `function_node.py` (around lines 1144–1163): - -```python - def _filter_by_content_hash(self, table: "pa.Table") -> "pa.Table": - """Filter *table* to rows whose ``NODE_CONTENT_HASH_COL`` matches this node. - - Only applied when ``table_scope="pipeline_hash"`` because in that mode - multiple runs share the same DB table and must be disambiguated at read - time. In ``"content_hash"`` mode every run has its own table so no - filtering is needed. - """ - if self._table_scope != "pipeline_hash": - return table - col_name = constants.NODE_CONTENT_HASH_COL - if col_name not in table.column_names: - raise ValueError( - f"Cannot isolate records for table_scope='pipeline_hash': " - f"required column {col_name!r} is missing from the stored table. " - "This may indicate records written by an older version of the code." - ) - own_hash = self.content_hash().to_prefixed_digest() - mask = pc.equal(table.column(col_name), own_hash) - return table.filter(mask) -``` - -Delete it entirely (20 lines). - -- [ ] **Step 2: Remove the `_filter_by_content_hash()` call from `_fetch_joined_records()`** - -Find this line in `_fetch_joined_records()` (around line 1915): - -```python - taginfo = self._filter_by_content_hash(taginfo) -``` - -Delete it. The two lines around it (`taginfo_columns = tuple(taginfo.column_names)` and `taginfo_schema = taginfo.schema`) stay. - -- [ ] **Step 3: Remove `NODE_CONTENT_HASH_COL` from the `drop_columns` list in `get_all_records()`** - -Find the `drop_columns = [...]` block in `get_all_records()` (around lines 1815–1820): - -```python - drop_columns = [ - constants.NODE_CONTENT_HASH_COL, - _PIPELINE_ENTRY_ID_COL, - _PIPELINE_BASE_ENTRY_ID_COL, - _PIPELINE_RECOMPUTATION_INDEX_COL, - ] -``` - -Remove `constants.NODE_CONTENT_HASH_COL,` so it becomes: - -```python - drop_columns = [ - _PIPELINE_ENTRY_ID_COL, - _PIPELINE_BASE_ENTRY_ID_COL, - _PIPELINE_RECOMPUTATION_INDEX_COL, - ] -``` - -Also update the docstring of `get_all_records()`. Replace: - -``` - ``_PIPELINE_ENTRY_ID_COL`` and - ``NODE_CONTENT_HASH_COL`` are always dropped — they are internal - discriminator columns, not user-facing data. -``` - -With: - -``` - ``_PIPELINE_ENTRY_ID_COL``, ``_PIPELINE_BASE_ENTRY_ID_COL``, and - ``_PIPELINE_RECOMPUTATION_INDEX_COL`` are always dropped — they are - internal discriminator columns, not user-facing data. -``` - -- [ ] **Step 4: Remove `pc` from the lazy-import block** - -`pc` (pyarrow.compute) was only used by `_filter_by_content_hash`. Remove it from both sides of the `TYPE_CHECKING` block (around lines 64 and 67): - -```python -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - import pyarrow.compute as pc # ← delete this line -else: - pa = LazyModule("pyarrow") - pc = LazyModule("pyarrow.compute") # ← delete this line - pl = LazyModule("polars") -``` - -Becomes: - -```python -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") -``` - -- [ ] **Step 5: Update the stale comment that mentions `_filter_by_content_hash()`** - -Find the comment block around line 1023 inside `FunctionJobNode.from_descriptor()`: - -```python - # match the _node_content_hash column written to the DB at run time. - # The blueprint hash stored in descriptor["content_hash"] is computed - # from schema-only upstreams and differs from the live hash, causing - # _filter_by_content_hash() to return zero rows in get_all_records(). - node._stored_content_hash = ( -``` - -Replace the comment with: - -```python - # Restore the live content hash from the serialised descriptor so that - # ``content_hash()`` on the deserialised node matches the hash that was - # computed when the original node ran. - node._stored_content_hash = ( -``` - -- [ ] **Step 6: Run the function_pod tests** - -```bash -uv run pytest tests/test_core/function_pod/ -v -``` - -Expected: all pass. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/core/nodes/function_node.py -git commit -m "refactor(function_node): delete _filter_by_content_hash, remove NODE_CONTENT_HASH_COL from get_all_records drop list" -``` - ---- - -### Task 5: Update `side_effects.py` - -**Files:** -- Modify: `src/orcapod/side_effects.py` -- Test: `tests/test_core/function_pod/test_node_content_hash_redundancy.py` (new test for tdb path) - -- [ ] **Step 1: Write failing tests for tdb versioning** - -Add a new test class at the end of `tests/test_core/function_pod/test_node_content_hash_redundancy.py`: - -```python -# --------------------------------------------------------------------------- -# tdb versioning: SideEffectJobNode must use TRACKING_DB_SCHEMA_VERSION suffix -# --------------------------------------------------------------------------- - - -class TestTdbVersioning: - """``SideEffectJobNode.attach_databases()`` must write to the tdb_v1 path.""" - - def test_tdb_v1_path_used_after_attach(self): - """After attaching a DB, ``_table_path`` ends with ``TRACKING_DB_SCHEMA_VERSION``.""" - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.databases import InMemoryArrowDatabase - from orcapod.side_effects import SideEffectJobNode, SideEffectPod - from orcapod.system_constants import TRACKING_DB_SCHEMA_VERSION - - table = pa.table({"x": pa.array([1], type=pa.int64())}) - source = ArrowTableSource(table=table, tag_columns=[], infer_nullable=True) - - def noop(x: int, ctx=None) -> None: - pass - - pod = SideEffectPod(fn=noop, ctx_arg_name="ctx") - node = SideEffectJobNode(side_effect_pod=pod, input_stream=source) - db = InMemoryArrowDatabase() - node.attach_databases(db) - - assert node._table_path is not None - assert node._table_path[-1] == TRACKING_DB_SCHEMA_VERSION, ( - f"Expected _table_path to end with {TRACKING_DB_SCHEMA_VERSION!r}; " - f"got: {node._table_path!r}" - ) - - def test_tdb_v0_entries_not_visible_to_new_code(self): - """Entries written at the bare (tdb_v0) path are not seen by the new code. - - ``SideEffectJobNode`` now reads/writes at ``...(tdb_v1,)`` — any entries - at the old bare path are orphaned and cause no error. - """ - import pyarrow as pa - from orcapod.core.sources.arrow_table_source import ArrowTableSource - from orcapod.databases import InMemoryArrowDatabase - from orcapod.side_effects import SideEffectJobNode, SideEffectPod - - table = pa.table({"x": pa.array([1], type=pa.int64())}) - source = ArrowTableSource(table=table, tag_columns=[], infer_nullable=True) - - def noop(x: int, ctx=None) -> None: - pass - - pod = SideEffectPod(fn=noop, ctx_arg_name="ctx") - node = SideEffectJobNode(side_effect_pod=pod, input_stream=source) - db = InMemoryArrowDatabase() - - # Write a fake "completed" entry at the OLD bare path (tdb_v0 location). - old_path = node.node_uri + (f"schema:{node.pipeline_hash().to_string()}",) - fake_record_id = b"\xde\xad" * 8 - fake_row = pa.table({"record_id_hash": pa.array(["old_hash"], type=pa.large_string())}) - db.add_record(old_path, fake_record_id, fake_row) - - # Attach — the new code uses the tdb_v1 path, not old_path. - node.attach_databases(db) - - # The new _table_path is different from old_path → old entry is orphaned. - assert node._table_path != old_path, ( - "New _table_path must differ from tdb_v0 bare path; old entries are orphaned." - ) - # No error raised — orphaned entries are silently ignored. -``` - -- [ ] **Step 2: Run the new tests to confirm they fail** - -```bash -uv run pytest "tests/test_core/function_pod/test_node_content_hash_redundancy.py::TestTdbVersioning" -v -``` - -Expected: `FAILED` (`_table_path` does not yet end with `"tdb_v1"`). - -- [ ] **Step 3: Add `TRACKING_DB_SCHEMA_VERSION` import to `side_effects.py`** - -In `src/orcapod/side_effects.py`, the existing import at the top is: - -```python -from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -``` - -Add after the last `from orcapod` import at the top of the file (before the `if TYPE_CHECKING:` block): - -```python -from orcapod.system_constants import TRACKING_DB_SCHEMA_VERSION -``` - -- [ ] **Step 4: Update `attach_databases()` to append `TRACKING_DB_SCHEMA_VERSION`** - -Find `attach_databases()` in `SideEffectJobNode` (around line 914). Replace: - -```python - self._pipeline_database = pipeline_database - if pipeline_database is not None: - self._table_path = self.node_uri + ( - f"schema:{self.pipeline_hash().to_string()}", - ) - else: - self._table_path = None -``` - -With: - -```python - self._pipeline_database = pipeline_database - if pipeline_database is not None: - self._table_path = self.node_uri + ( - f"schema:{self.pipeline_hash().to_string()}", - TRACKING_DB_SCHEMA_VERSION, - ) - else: - self._table_path = None -``` - -Also update the docstring of `attach_databases()`. Replace: - -``` - The table path is - ``self.node_uri + (f"schema:{self.pipeline_hash().to_string()}",)`` — - the same scoping convention used by ``FunctionNode`` and - ``OperatorNode``. -``` - -With: - -``` - The table path is - ``self.node_uri + (f"schema:{self.pipeline_hash().to_string()}", TRACKING_DB_SCHEMA_VERSION)`` - — scoped by pipeline hash and tdb schema version. -``` - -- [ ] **Step 5: Run tdb tests — should now pass** - -```bash -uv run pytest "tests/test_core/function_pod/test_node_content_hash_redundancy.py::TestTdbVersioning" -v -``` - -Expected: `PASSED` - -- [ ] **Step 6: Update `_execute_side_effect_row()` — remove `node_content_hash_str`** - -Find `_execute_side_effect_row()` (around line 294). - -Remove `node_content_hash_str: str` from the parameter list: - -```python -def _execute_side_effect_row( - *, - fn: Callable, - tag: TagProtocol, - data: DataProtocol, - pod_config: SideEffectPodConfig, - pipeline_hash_ch: ContentHash, - node_content_hash_str: str, # ← delete this line - pod_name: str, - run_id: str | None, - arrow_hasher: Any, - ctx_arg_name: str = "ctx", - pipeline_database: ArrowDatabaseProtocol | None = None, - table_path: tuple[str, ...] | None = None, -) -> tuple[TagProtocol, DataProtocol] | None: -``` - -Replace the preimage construction block (the `preimage = (tag.as_table(...) ...` block, around lines 348–362) with: - -```python - from orcapod.core.nodes.function_node import _build_record_id_preimage - - # 1. Build the preimage — delegates to the shared helper used by FunctionJobNode. - # Appends the recomputation index (fixed at 0; side effects never recompute). - preimage = _build_record_id_preimage(tag, data).append_column( - _SIDE_EFFECT_RECOMPUTATION_INDEX_COL, - pa.array([0], type=pa.int32()), - ) -``` - -Update the docstring of `_execute_side_effect_row()`. Remove the `node_content_hash_str` parameter description and remove its mention from the preimage description. Replace: - -``` - Computes a deterministic ``record_id`` from a preimage that matches - ``FunctionNode._build_entry_id_preimage`` plus a recomputation index of - ``0``. The preimage covers: - - * tag system-tag columns, - * ``INPUT_DATA_HASH_COL`` — ``data.content_hash().to_string()``, - * ``NODE_CONTENT_HASH_COL`` (the pod's own content hash), and - * ``_SIDE_EFFECT_RECOMPUTATION_INDEX_COL`` fixed at ``0`` (side effects - never recompute). -``` - -With: - -``` - Computes a deterministic ``record_id`` from a preimage built by - ``_build_record_id_preimage(tag, data)`` plus a fixed recomputation index - of ``0``. The preimage covers: - - * tag system-tag columns, - * ``INPUT_DATA_HASH_COL`` (binary ``data.content_hash().to_prefixed_digest()``), and - * ``_SIDE_EFFECT_RECOMPUTATION_INDEX_COL`` fixed at ``0`` (side effects - never recompute). -``` - -Also remove the `node_content_hash_str` entry from the Args section: - -``` - node_content_hash_str: ``pod.content_hash().to_string()`` — included in - the preimage as ``NODE_CONTENT_HASH_COL`` to scope the record ID - to this specific pod version. -``` - -- [ ] **Step 7: Remove `node_content_hash_str=...` from the 4 call sites** - -There are 4 calls to `_execute_side_effect_row`. In each, delete the `node_content_hash_str=self._pod.content_hash().to_string(),` kwarg line. - -**Call site 1** — `SideEffectPodStream.iter_data()` (around line 242): -```python - result = _execute_side_effect_row( - fn=self._pod._fn, - tag=tag, - data=data, - pod_config=self._pod.pod_config, - pipeline_hash_ch=self.pipeline_hash(), - node_content_hash_str=self._pod.content_hash().to_string(), # ← delete - pod_name=self._pod.label, - run_id=None, - arrow_hasher=self._pod.data_context.arrow_hasher, -``` - -**Call site 2** — `SideEffectNode.iter_data()` (around line 822): -Same pattern — delete the `node_content_hash_str=...` line. - -**Call site 3** — `SideEffectJobNode.execute()` (around line 960): -Same pattern — delete the `node_content_hash_str=...` line. - -**Call site 4** — async path inside `SideEffectJobNode` (around line 1020): -Same pattern — delete the `node_content_hash_str=...` line. - -- [ ] **Step 8: Run side-effect tests** - -```bash -uv run pytest tests/test_core/side_effect_pod/ tests/test_core/side_effect_function/ -v -``` - -Expected: all pass. - -- [ ] **Step 9: Run full redundancy test file** - -```bash -uv run pytest tests/test_core/function_pod/test_node_content_hash_redundancy.py -v -``` - -Expected: all pass (including the two new `TestTdbVersioning` tests). - -- [ ] **Step 10: Commit** - -```bash -git add src/orcapod/side_effects.py \ - tests/test_core/function_pod/test_node_content_hash_redundancy.py -git commit -m "feat(side_effects): remove node_content_hash_str from preimage, use tdb_v1 path" -``` - ---- - -### Task 6: Extend pdb migration - -**Files:** -- Modify: `src/orcapod/migrations/pipeline_db.py` -- Modify: `tests/test_migrations/test_pipeline_db.py` - -- [ ] **Step 1: Update existing migration tests that will break** - -In `tests/test_migrations/test_pipeline_db.py`, two tests currently assert that `NODE_CONTENT_HASH_COL` is present in v1. Update them to assert it is **absent**. - -**`test_v1_hash_columns_are_binary`**: replace the assertion that checks `NODE_CONTENT_HASH_COL` type: - -Old: -```python - assert v1_table.schema.field(constants.NODE_CONTENT_HASH_COL).type == pa.large_binary() - assert v1_table.schema.field(constants.INPUT_DATA_HASH_COL).type == pa.large_binary() - assert v1_table.schema.field(constants.OUTPUT_DATA_HASH_COL).type == pa.large_binary() -``` - -New: -```python - assert constants.NODE_CONTENT_HASH_COL not in v1_table.schema.names, ( - "__node_content_hash must be dropped from pdb_v1 rows." - ) - assert v1_table.schema.field(constants.INPUT_DATA_HASH_COL).type == pa.large_binary() - assert v1_table.schema.field(constants.OUTPUT_DATA_HASH_COL).type == pa.large_binary() -``` - -**`test_v1_binary_values_decode_correctly`**: remove the `NODE_CONTENT_HASH_COL` decode assertion: - -Old: -```python - assert ContentHash.from_prefixed_digest(bytes(row_dict[constants.NODE_CONTENT_HASH_COL])) == _NODE_HASH - assert ContentHash.from_prefixed_digest(bytes(row_dict[constants.INPUT_DATA_HASH_COL])) == _INPUT_HASH - assert ContentHash.from_prefixed_digest(bytes(row_dict[constants.OUTPUT_DATA_HASH_COL])) == _OUTPUT_HASH -``` - -New: -```python - assert constants.NODE_CONTENT_HASH_COL not in row_dict, ( - "__node_content_hash must be absent from migrated v1 rows." - ) - assert ContentHash.from_prefixed_digest(bytes(row_dict[constants.INPUT_DATA_HASH_COL])) == _INPUT_HASH - assert ContentHash.from_prefixed_digest(bytes(row_dict[constants.OUTPUT_DATA_HASH_COL])) == _OUTPUT_HASH -``` - -- [ ] **Step 2: Run the modified existing tests — confirm they fail** - -```bash -uv run pytest tests/test_migrations/test_pipeline_db.py::TestMigratePipelineV0ToV1::test_v1_hash_columns_are_binary \ - tests/test_migrations/test_pipeline_db.py::TestMigratePipelineV0ToV1::test_v1_binary_values_decode_correctly -v -``` - -Expected: `FAILED` — `NODE_CONTENT_HASH_COL` is currently still written. - -- [ ] **Step 3: Add a helper for v0 rows with system tags + add new failing tests** - -Add at the top of `tests/test_migrations/test_pipeline_db.py`, after the existing `_write_v0_pdb_row` helper: - -```python -_SYS_TAG_COL = f"{constants.SYSTEM_TAG_PREFIX}source_id::deadbeef" -_SYS_TAG_VAL = "source_abc" - - -def _write_v0_pdb_row_with_system_tags( - db: InMemoryArrowDatabase, - pdb_path: tuple, - data_id: bytes, - pdb_record_id: bytes, -) -> None: - """Write a v0 pdb row that includes a system-tag column. - - Used to test that the extended migration correctly identifies system-tag - columns and includes them in the recomputed preimage. - """ - row = pa.table({ - _SYS_TAG_COL: pa.array([_SYS_TAG_VAL], type=pa.large_string()), - constants.DATA_RECORD_ID: pa.array([data_id], type=pa.large_binary()), - constants.NODE_CONTENT_HASH_COL: pa.array([_NODE_HASH.to_string()], type=pa.large_string()), - constants.INPUT_DATA_HASH_COL: pa.array([_INPUT_HASH.to_string()], type=pa.large_string()), - constants.OUTPUT_DATA_HASH_COL: pa.array([_OUTPUT_HASH.to_string()], type=pa.large_string()), - f"{constants.META_PREFIX}input_data{constants.CONTEXT_KEY}": pa.array(["ctx"], type=pa.large_string()), - f"{constants.META_PREFIX}computed": pa.array([True], type=pa.bool_()), - constants.IS_EPHEMERAL_COL: pa.array([False], type=pa.bool_()), - "__pipeline_base_entry_id": pa.array([b"old_base"], type=pa.large_binary()), - "__pipeline_recomputation_index": pa.array([0], type=pa.int32()), - }) - db.add_record(pdb_path, pdb_record_id, row) - db.flush() -``` - -Then add a new test class at the end of the file: - -```python -class TestMigratePipelineV0ToV1Extended: - """Tests for the ITL-533 extensions to migrate_pipeline_v0_to_v1.""" - - def test_migrate_v0_to_v1_drops_node_content_hash(self): - """Migrated v1 rows must not contain ``__node_content_hash``.""" - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - - _write_v0_pdb_row_with_system_tags(db, pdb_path, _DATA_ID, _PDB_RECORD_ID) - - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - assert v1_table is not None - assert constants.NODE_CONTENT_HASH_COL not in v1_table.schema.names, ( - "``__node_content_hash`` must be dropped from pdb_v1 rows." - ) - - def test_migrate_v0_to_v1_recomputes_base_entry_id(self): - """``__pipeline_base_entry_id`` is recomputed using only system_tags + INPUT_DATA_HASH_COL.""" - import pyarrow as pa - from orcapod.hashing.defaults import get_default_arrow_hasher - - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - - _write_v0_pdb_row_with_system_tags(db, pdb_path, _DATA_ID, _PDB_RECORD_ID) - - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - assert v1_table is not None - row = v1_table.to_pylist()[0] - - # Independently compute the expected base_entry_id. - arrow_hasher = get_default_arrow_hasher() - preimage = pa.table({ - _SYS_TAG_COL: pa.array([_SYS_TAG_VAL], type=pa.large_string()), - constants.INPUT_DATA_HASH_COL: pa.array( - [_INPUT_HASH.to_prefixed_digest()], type=pa.large_binary() - ), - }) - expected = arrow_hasher.hash_table(preimage).to_prefixed_digest() - - actual = bytes(row["__pipeline_base_entry_id"]) - assert actual == expected, ( - f"Recomputed base_entry_id mismatch.\n" - f"Expected: {expected!r}\n" - f"Actual: {actual!r}" - ) - - def test_migrate_v0_to_v1_recomputes_record_id(self): - """``__record_id`` is recomputed using system_tags + INPUT_DATA_HASH_COL + recomputation_index.""" - import pyarrow as pa - from orcapod.hashing.defaults import get_default_arrow_hasher - - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - - _write_v0_pdb_row_with_system_tags(db, pdb_path, _DATA_ID, _PDB_RECORD_ID) - - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path, record_id_column="__record_id") - assert v1_table is not None - row = v1_table.to_pylist()[0] - - arrow_hasher = get_default_arrow_hasher() - preimage = pa.table({ - _SYS_TAG_COL: pa.array([_SYS_TAG_VAL], type=pa.large_string()), - constants.INPUT_DATA_HASH_COL: pa.array( - [_INPUT_HASH.to_prefixed_digest()], type=pa.large_binary() - ), - }) - preimage_with_idx = preimage.append_column( - "__pipeline_recomputation_index", - pa.array([0], type=pa.int32()), - ) - expected = arrow_hasher.hash_table(preimage_with_idx).to_prefixed_digest() - - actual = bytes(row["__record_id"]) - assert actual == expected, ( - f"Recomputed record_id mismatch.\n" - f"Expected: {expected!r}\n" - f"Actual: {actual!r}" - ) - - def test_migrate_v0_to_v1_extended_idempotent(self): - """Running the extended migration twice produces no duplicate v1 rows.""" - db = InMemoryArrowDatabase() - pdb_path = ("pipeline",) - rdb_path = ("results",) - - _write_v0_pdb_row_with_system_tags(db, pdb_path, _DATA_ID, _PDB_RECORD_ID) - - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - migrate_pipeline_v0_to_v1(db, pdb_path, db, rdb_path, progress=False) - - v1_path = pdb_path + (PIPELINE_DB_SCHEMA_VERSION,) - v1_table = db.get_all_records(v1_path) - assert v1_table is not None - assert v1_table.num_rows == 1, ( - f"Expected exactly 1 v1 row after two migration runs; got {v1_table.num_rows}." - ) -``` - -- [ ] **Step 4: Run all migration tests — confirm new tests fail** - -```bash -uv run pytest tests/test_migrations/test_pipeline_db.py -v -``` - -Expected: `test_v1_hash_columns_are_binary` FAIL, `test_v1_binary_values_decode_correctly` FAIL, all 4 new `TestMigratePipelineV0ToV1Extended` tests FAIL, others PASS. - -- [ ] **Step 5: Implement the extended migration in `pipeline_db.py`** - -**5a. Update `_PDB_HASH_COLS`** — remove `NODE_CONTENT_HASH_COL` (it is dropped from v1, no longer converted): - -```python -# pdb columns whose values are ContentHash strings in v0 and must become binary in v1. -# NODE_CONTENT_HASH_COL is intentionally excluded — it is dropped during migration. -_PDB_HASH_COLS = frozenset({ - constants.INPUT_DATA_HASH_COL, - constants.OUTPUT_DATA_HASH_COL, -}) -``` - -**5b. Add `get_default_arrow_hasher` import** at the top of `pipeline_db.py`, after the existing imports: - -```python -from orcapod.hashing.defaults import get_default_arrow_hasher -``` - -**5c. Acquire the arrow hasher in `migrate_pipeline_v0_to_v1()`** — add one line before the batch loop: - -```python - rows_migrated = 0 - rows_skipped = 0 - rows_unresolvable = 0 - arrow_hasher = get_default_arrow_hasher() # ← add this line - - for batch_start in range(0, rows_total, batch_size): -``` - -**5d. Pass `arrow_hasher` to `_transform_pdb_batch()`** — update the call: - -```python - transformed, batch_unresolvable = _transform_pdb_batch(new_rows, rdb_index, arrow_hasher) -``` - -**5e. Update `_transform_pdb_batch()` signature and body**: - -Replace the full function with: - -```python -def _transform_pdb_batch( - batch: pa.Table, - rdb_index: dict[bytes, dict], - arrow_hasher: Any, -) -> tuple[pa.Table, int]: - """Transform a batch of v0 pdb rows into v1 format. - - Per-row transformations (applied in order): - - 1. Drop ``__node_content_hash`` — not stored in v1. - 2. Convert ``__input_data_hash`` from ``large_string`` → ``large_binary`` - (falls back to rdb index when the pdb value is ``None``). - 3. Convert ``__output_data_hash`` from ``large_string`` → ``large_binary`` - (no rdb fallback; ``None`` stays ``null``). - 4. Recompute ``__pipeline_base_entry_id`` and ``__record_id`` using the new - preimage (``system_tag_cols + INPUT_DATA_HASH_COL``, without - ``NODE_CONTENT_HASH_COL``). Rows where ``__input_data_hash`` cannot be - resolved are counted as unresolvable and their hash columns are left - ``null``. - - Args: - batch: Arrow table slice of v0 pdb rows (with ``_RECORD_ID_COL`` as first - column, as returned by ``get_all_records(record_id_column=...)``). - rdb_index: Dict mapping rdb record-ID bytes to row dicts. - arrow_hasher: ``ArrowHasherProtocol`` used for recomputing the hash columns. - - Returns: - Tuple of (transformed Arrow table, count of unresolvable rows). - """ - node_hash_col = constants.NODE_CONTENT_HASH_COL - input_hash_col = constants.INPUT_DATA_HASH_COL - output_hash_col = constants.OUTPUT_DATA_HASH_COL - data_id_col = constants.DATA_RECORD_ID - sys_tag_prefix = constants.SYSTEM_TAG_PREFIX - - rows = batch.to_pylist() - unresolvable = 0 - out_rows: list[dict] = [] - - for row in rows: - new_row = dict(row) - - # 1. Drop __node_content_hash — not stored in v1. - new_row.pop(node_hash_col, None) - - # 2. Convert __input_data_hash from string → binary. - val = new_row.get(input_hash_col) - if val is not None and isinstance(val, str): - new_row[input_hash_col] = ContentHash.from_string(val).to_prefixed_digest() - elif val is None: - # Missing in pdb — fall back to rdb index. - data_id = row.get(data_id_col) - if data_id is not None: - data_id_bytes = bytes(data_id) - rdb_row = rdb_index.get(data_id_bytes) - if rdb_row is not None: - raw = rdb_row.get(input_hash_col) - if raw is not None and isinstance(raw, str): - new_row[input_hash_col] = ContentHash.from_string(raw).to_prefixed_digest() - else: - unresolvable += 1 - - # 3. Convert __output_data_hash from string → binary. - val = new_row.get(output_hash_col) - if val is not None and isinstance(val, str): - new_row[output_hash_col] = ContentHash.from_string(val).to_prefixed_digest() - - # 4. Recompute __pipeline_base_entry_id and __record_id using new preimage. - input_hash_bytes = new_row.get(input_hash_col) - if input_hash_bytes is not None: - sys_tag_cols = sorted(c for c in row if c.startswith(sys_tag_prefix)) - preimage_arrays: dict[str, pa.Array] = {} - for col in sys_tag_cols: - preimage_arrays[col] = pa.array([row.get(col)], type=pa.large_string()) - preimage_arrays[input_hash_col] = pa.array( - [input_hash_bytes], type=pa.large_binary() - ) - preimage = pa.table(preimage_arrays) - - new_base_entry_id = arrow_hasher.hash_table(preimage).to_prefixed_digest() - new_row["__pipeline_base_entry_id"] = new_base_entry_id - - recomp_idx = new_row.get("__pipeline_recomputation_index") or 0 - preimage_with_idx = preimage.append_column( - "__pipeline_recomputation_index", - pa.array([recomp_idx], type=pa.int32()), - ) - new_row[_RECORD_ID_COL] = arrow_hasher.hash_table(preimage_with_idx).to_prefixed_digest() - - out_rows.append(new_row) - - transformed = pa.Table.from_pylist(out_rows, schema=_v1_pdb_schema(batch)) - return transformed, unresolvable -``` - -**5f. Update `_v1_pdb_schema()` to exclude `NODE_CONTENT_HASH_COL`**: - -```python -def _v1_pdb_schema(v0_batch: pa.Table) -> pa.Schema: - """Derive the v1 pdb Arrow schema from a v0 batch. - - Drops ``__node_content_hash`` (removed in pdb_v1) and replaces the two - remaining ContentHash columns (``__input_data_hash``, ``__output_data_hash``) - with ``large_binary`` equivalents. All other columns retain their original - types. - - Args: - v0_batch: A v0 pdb Arrow table (used to read non-hash column types). - - Returns: - Arrow schema for the v1 pdb table. - """ - node_hash_col = constants.NODE_CONTENT_HASH_COL - fields = [] - for field in v0_batch.schema: - if field.name == node_hash_col: - continue # Drop __node_content_hash from v1. - if field.name in _PDB_HASH_COLS: - fields.append(pa.field(field.name, pa.large_binary(), nullable=True)) - else: - fields.append(field) - # Ensure the remaining hash columns exist even if absent in v0. - existing_names = {f.name for f in fields} - for col in _PDB_HASH_COLS: - if col not in existing_names: - fields.append(pa.field(col, pa.large_binary(), nullable=True)) - return pa.schema(fields) -``` - -- [ ] **Step 6: Run all migration tests — all must pass** - -```bash -uv run pytest tests/test_migrations/test_pipeline_db.py -v -``` - -Expected: all pass (both updated existing tests and all 4 new tests). - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/migrations/pipeline_db.py \ - tests/test_migrations/test_pipeline_db.py -git commit -m "feat(migrations): extend pdb v0→v1 to drop __node_content_hash and recompute hash columns" -``` - ---- - -### Task 7: Full suite verification - -- [ ] **Step 1: Run the entire test suite** - -```bash -uv run pytest tests/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: all tests pass. Key suites to watch: -- `tests/test_core/function_pod/` — all pass -- `tests/test_core/side_effect_pod/` — all pass -- `tests/test_core/side_effect_function/` — all pass -- `tests/test_migrations/` — all pass -- `tests/test_core/function_pod/test_node_content_hash_redundancy.py` — all 14 pass (was 12; 2 added in Task 3 + 2 `TestTdbVersioning`) - -- [ ] **Step 2: If any test fails, diagnose and fix before committing** - -For `function_pod` failures: check for any remaining `NODE_CONTENT_HASH_COL` reference in assertion messages or expected schema shapes. - -For `side_effect_pod/side_effect_function` failures: check that all 4 call sites to `_execute_side_effect_row()` have `node_content_hash_str` removed. - -For migration failures: check `_transform_pdb_batch` handles rows that have no system_tag columns (the existing test fixtures use rows without them — in that case `sys_tag_cols` is empty and the preimage is just `{INPUT_DATA_HASH_COL: ...}`). - -- [ ] **Step 3: Final commit (if any fixes were needed)** - -```bash -git add -p # stage only the fix hunks -git commit -m "fix: address full-suite failures after NODE_CONTENT_HASH_COL removal" -``` - ---- - -## Self-review - -**Spec coverage check:** - -| Spec requirement | Task | -|---|---| -| `_build_entry_id_preimage()` returns only `system_tags + INPUT_DATA_HASH_COL` | Task 2 | -| `_execute_side_effect_row()` uses same preimage; `node_content_hash_str` removed | Task 5 | -| `add_pipeline_record()` no longer writes `__node_content_hash` | Task 3 | -| `_filter_by_content_hash()` deleted; call site removed | Task 4 | -| `get_all_records()` stops referencing `__node_content_hash` | Task 4 | -| Shared `_build_record_id_preimage()` helper extracted | Task 2 | -| `PIPELINE_DB_SCHEMA_VERSION` remains `"pdb_v1"` | No change needed | -| `TRACKING_DB_SCHEMA_VERSION = "tdb_v1"` added | Task 1 | -| `SideEffectJobNode` appends `TRACKING_DB_SCHEMA_VERSION` to `_table_path` | Task 5 | -| `migrate_pipeline_v0_to_v1()` drops `__node_content_hash` and recomputes hashes | Task 6 | -| `test_node_content_hash_col_not_in_preimage_keys` turns green | Task 2 | -| New migration tests: drop, recompute_base, recompute_record, idempotent | Task 6 | -| `test_tdb_v1_path_used`, `test_tdb_v0_entries_orphaned` | Task 5 | - -All spec requirements covered. ✓ diff --git a/superpowers/plans/2026-07-21-itl-545-release-branch-argument.md b/superpowers/plans/2026-07-21-itl-545-release-branch-argument.md deleted file mode 100644 index 951a45e89..000000000 --- a/superpowers/plans/2026-07-21-itl-545-release-branch-argument.md +++ /dev/null @@ -1,423 +0,0 @@ -# Release Branch Argument Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add an optional `branch` input (default `main`) to the GitHub Actions release workflow so releases can be cut from any branch, with a pre-flight validation step that fails fast on invalid branch names. - -**Architecture:** Two files change. `release.yml` gets a new `branch` workflow input, a new `validate-branch` pre-flight job, and updated checkout steps in `test` and `build`. `RELEASING.md` gets updated docs and a new hotfix-release example. - -**Tech Stack:** GitHub Actions YAML, `gh` CLI (pre-installed on ubuntu-latest runners), PyYAML (for local syntax validation), Bash - ---- - -## File Map - -| Action | File | What changes | -|--------|------|--------------| -| Modify | `.github/workflows/release.yml` | Add `branch` input; add `validate-branch` job; update `test`, `license-check`, `build` jobs | -| Modify | `RELEASING.md` | Update Step 2; add "Cutting a Hotfix Release" section | - ---- - -### Task 1: Create the feature branch - -- [ ] **Step 1: Check out the feature branch** - -```bash -git checkout -b eywalker/itl-545-release-cutting-allow-optional-branch-argument-defaults-to -``` - -- [ ] **Step 2: Verify you are on the correct branch** - -```bash -git branch --show-current -``` - -Expected output: -``` -eywalker/itl-545-release-cutting-allow-optional-branch-argument-defaults-to -``` - ---- - -### Task 2: Add `branch` input to `release.yml` - -**Files:** -- Modify: `.github/workflows/release.yml:1-9` - -The `workflow_dispatch.inputs` block currently has only `version`. Add `branch` immediately after it. - -- [ ] **Step 1: Edit the `on:` block** - -Replace the current `on:` block (lines 1–9): - -```yaml -name: Release - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g. 0.1.0 or v0.1.0 — leading v is stripped automatically)' - required: true - type: string -``` - -With: - -```yaml -name: Release - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g. 0.1.0 or v0.1.0 — leading v is stripped automatically)' - required: true - type: string - branch: - description: 'Branch to cut the release from (defaults to main)' - required: false - type: string - default: main -``` - -- [ ] **Step 2: Validate YAML syntax** - -```bash -uv run python -c "import yaml; yaml.safe_load(open('.github/workflows/release.yml')); print('YAML OK')" -``` - -Expected output: -``` -YAML OK -``` - -- [ ] **Step 3: Commit** - -```bash -git add .github/workflows/release.yml -git commit -m "ci(release): add optional branch input (defaults to main)" -``` - ---- - -### Task 3: Add `validate-branch` pre-flight job - -**Files:** -- Modify: `.github/workflows/release.yml` — insert new job before `test` - -The new job uses the built-in `GITHUB_TOKEN` (exposed as `GH_TOKEN` env var) to query the GitHub API. No checkout needed. `gh` is pre-installed on all `ubuntu-latest` runners. - -- [ ] **Step 1: Insert the `validate-branch` job** - -In `.github/workflows/release.yml`, after the closing `type: string` line of the `branch` input and before the `jobs:` key, the file structure is: - -```yaml -jobs: - test: - name: Test ... -``` - -Add the `validate-branch` job as the **first** job under `jobs:`: - -```yaml -jobs: - validate-branch: - name: Validate release branch - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Check branch exists on origin - env: - GH_TOKEN: ${{ github.token }} - run: | - BRANCH="${{ inputs.branch }}" - if ! gh api "repos/${{ github.repository }}/branches/${BRANCH}" --silent 2>/dev/null; then - echo "::error::Branch '${BRANCH}' does not exist. Check for typos." - exit 1 - fi - echo "Branch '${BRANCH}' confirmed." - - test: - name: Test ... -``` - -- [ ] **Step 2: Validate YAML syntax** - -```bash -uv run python -c "import yaml; yaml.safe_load(open('.github/workflows/release.yml')); print('YAML OK')" -``` - -Expected output: -``` -YAML OK -``` - -- [ ] **Step 3: Commit** - -```bash -git add .github/workflows/release.yml -git commit -m "ci(release): add validate-branch pre-flight job" -``` - ---- - -### Task 4: Wire `branch` into `test` and `license-check` jobs - -**Files:** -- Modify: `.github/workflows/release.yml` — `test` and `license-check` jobs - -Both jobs must now depend on `validate-branch`. The `test` job's checkout must use `ref: ${{ inputs.branch }}` so tests run against the release branch. - -- [ ] **Step 1: Update the `test` job** - -Find the `test` job. It currently starts with: - -```yaml - test: - name: Test (Python ${{ matrix.python-version }}) - runs-on: ubuntu-latest -``` - -And its checkout step is: - -```yaml - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 # required: hatch-vcs needs full tag history -``` - -Make two changes: - -1. Add `needs: [validate-branch]` between `name:` and `runs-on:`: - -```yaml - test: - name: Test (Python ${{ matrix.python-version }}) - needs: [validate-branch] - runs-on: ubuntu-latest -``` - -2. Add `ref: ${{ inputs.branch }}` to the checkout `with:` block: - -```yaml - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 # required: hatch-vcs needs full tag history - ref: ${{ inputs.branch }} -``` - -- [ ] **Step 2: Update the `license-check` job** - -Find the `license-check` job. It currently reads: - -```yaml - license-check: - uses: ./.github/workflows/_license-check.yml -``` - -Add `needs: [validate-branch]`: - -```yaml - license-check: - needs: [validate-branch] - uses: ./.github/workflows/_license-check.yml -``` - -- [ ] **Step 3: Validate YAML syntax** - -```bash -uv run python -c "import yaml; yaml.safe_load(open('.github/workflows/release.yml')); print('YAML OK')" -``` - -Expected output: -``` -YAML OK -``` - -- [ ] **Step 4: Commit** - -```bash -git add .github/workflows/release.yml -git commit -m "ci(release): wire branch input into test and license-check jobs" -``` - ---- - -### Task 5: Wire `branch` into `build` job - -**Files:** -- Modify: `.github/workflows/release.yml` — `build` job's checkout step - -The `build` job creates the git tag and pushes it. Its checkout must use `ref: ${{ inputs.branch }}` so the tag is created at the tip of the release branch, not at whatever commit triggered the dispatch. - -- [ ] **Step 1: Update the `build` job checkout** - -Find the checkout step inside `build`. It currently reads: - -```yaml - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 # required: hatch-vcs reads tag to set version -``` - -Add `ref: ${{ inputs.branch }}`: - -```yaml - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 # required: hatch-vcs reads tag to set version - ref: ${{ inputs.branch }} -``` - -- [ ] **Step 2: Validate YAML syntax** - -```bash -uv run python -c "import yaml; yaml.safe_load(open('.github/workflows/release.yml')); print('YAML OK')" -``` - -Expected output: -``` -YAML OK -``` - -- [ ] **Step 3: Verify the full job graph** - -Confirm the structure of `release.yml` looks like this (just the job names and `needs:` lines — spot-check): - -```bash -uv run python -c " -import yaml -wf = yaml.safe_load(open('.github/workflows/release.yml')) -for name, job in wf['jobs'].items(): - needs = job.get('needs', []) - print(f'{name}: needs={needs}') -" -``` - -Expected output: -``` -validate-branch: needs=[] -test: needs=['validate-branch'] -license-check: needs=['validate-branch'] -build: needs=['test', 'license-check'] -publish-testpypi: needs=['build'] -publish-pypi: needs=['build', 'publish-testpypi'] -linear-sync: needs=['build'] -linear-complete: needs=['build', 'publish-pypi', 'linear-sync'] -``` - -- [ ] **Step 4: Commit** - -```bash -git add .github/workflows/release.yml -git commit -m "ci(release): wire branch input into build checkout" -``` - ---- - -### Task 6: Update `RELEASING.md` - -**Files:** -- Modify: `RELEASING.md` - -Two edits: - -1. **Step 2 of "Cutting a Release"** — add a note about the optional branch field. -2. **New "Cutting a Hotfix Release" section** — insert after the existing "Pre-releases" section. - -- [ ] **Step 1: Update Step 2 of "Cutting a Release"** - -Find this paragraph in `RELEASING.md`: - -```markdown -2. **Trigger the release workflow** — go to - **[Actions → Release → Run workflow](https://github.com/nauticalab/orcapod-python/actions/workflows/release.yml)** - in the GitHub UI, enter the version (e.g. `0.1.0`), and click **Run workflow**. - - A leading `v` is stripped automatically — `v0.1.0` and `0.1.0` both work. -``` - -Replace it with: - -```markdown -2. **Trigger the release workflow** — go to - **[Actions → Release → Run workflow](https://github.com/nauticalab/orcapod-python/actions/workflows/release.yml)** - in the GitHub UI, enter the version (e.g. `0.1.0`), optionally enter a branch name - (defaults to `main`), and click **Run workflow**. - - A leading `v` is stripped automatically — `v0.1.0` and `0.1.0` both work. -``` - -- [ ] **Step 2: Add "Cutting a Hotfix Release" section** - -Find the "Pre-releases" section header in `RELEASING.md`: - -```markdown -## Pre-releases -``` - -Insert the following new section **after** the entire "Pre-releases" section (i.e., after the last paragraph of "Pre-releases", before "## Tag Format"): - -```markdown -## Cutting a Hotfix Release - -To release from a branch other than `main` — for example, a maintenance branch carrying -a critical patch — enter the branch name in the **Branch** field when triggering the -workflow. - -**Example:** cutting `v0.1.1` from `hotfix/0.1.x`: - -1. **Trigger the release workflow** — go to - **[Actions → Release → Run workflow](https://github.com/nauticalab/orcapod-python/actions/workflows/release.yml)** -2. Set **Version** to `0.1.1` -3. Set **Branch** to `hotfix/0.1.x` -4. Click **Run workflow** - -The workflow will confirm the branch exists, run tests against it, create the `v0.1.1` -tag at its tip, and publish to PyPI — identical to a normal release. - -``` - -- [ ] **Step 3: Commit** - -```bash -git add RELEASING.md -git commit -m "docs(releasing): document optional branch argument and hotfix example" -``` - ---- - -### Task 7: Push branch and open PR - -- [ ] **Step 1: Push the feature branch** - -```bash -git push -u origin eywalker/itl-545-release-cutting-allow-optional-branch-argument-defaults-to -``` - -- [ ] **Step 2: Open the PR** - -```bash -gh pr create \ - --title "ci(release): allow optional branch argument (defaults to main)" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Adds an optional `branch` input to the `release.yml` workflow dispatch (defaults to `main`) -- Adds a `validate-branch` pre-flight job that fails fast with a clear error if the branch doesn't exist -- Wires `ref: ${{ inputs.branch }}` into the `test` and `build` checkout steps so tests and tag creation target the release branch -- Updates `RELEASING.md` with the new parameter and a hotfix release example - -Closes ITL-545 - -## Test plan - -- [ ] Trigger the release workflow without specifying a branch — confirm behavior is identical to today (cuts from `main`) -- [ ] Trigger the release workflow with a valid non-main branch name — confirm the tag is created at the tip of that branch -- [ ] Trigger the release workflow with a non-existent branch name — confirm `validate-branch` fails immediately with the error message `Branch '...' does not exist. Check for typos.` and no other jobs run -EOF -)" -``` diff --git a/superpowers/plans/2026-07-21-postrunpayload-invocation-context.md b/superpowers/plans/2026-07-21-postrunpayload-invocation-context.md deleted file mode 100644 index a76de822c..000000000 --- a/superpowers/plans/2026-07-21-postrunpayload-invocation-context.md +++ /dev/null @@ -1,810 +0,0 @@ -# PostRunPayload InvocationContext Alignment Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `invocation_context: InvocationContext | None` to `PostRunPayload` so post-run hook authors have access to the same deterministic invocation hash, `format_id()`, and `pipeline_run_id` that `SideEffectPod` functions already receive. - -**Architecture:** Extract `InvocationContext`/`InvocationHashConfig` from `side_effects.py` into a new stdlib-only leaf module `invocation.py`. Add the field to `PostRunPayload` (with `= None` default for backward compat). Thread `run_id` into `_build_post_run_payload` and call `_build_invocation_context()` there so every hook payload carries a fully-populated context. - -**Tech Stack:** Python 3.11+, PyArrow (for preimage hashing in `_build_invocation_context`), pytest + uv. - -**Issue:** ITL-531 -**Spec:** `superpowers/specs/2026-07-21-postrunpayload-invocation-context-design.md` -**Branch:** `eywalker/itl-531-align-postrunpayload-with-sideeffectpod-invocationcontext` - ---- - -## File Map - -| Action | File | Purpose | -|--------|------|---------| -| **Create** | `src/orcapod/invocation.py` | New leaf module: `InvocationHashConfig`, `_serialize_component`, `InvocationContext` | -| **Modify** | `src/orcapod/side_effects.py` | Remove moved definitions; import from `invocation.py` | -| **Modify** | `src/orcapod/hooks.py` | Import `InvocationContext`; add `invocation_context` field to `PostRunPayload` | -| **Modify** | `src/orcapod/core/function_pod.py` | Thread `run_id` into `_build_post_run_payload`; build and attach `InvocationContext` | -| **Modify** | `src/orcapod/core/cached_function_pod.py` | Pass `run_id` to `_build_post_run_payload` in overridden hook methods | -| **Modify** | `src/orcapod/__init__.py` | Import `InvocationContext`/`InvocationHashConfig` from `.invocation` instead of `.side_effects` | -| **Modify** | `tests/test_core/function_pod/test_post_run_hooks.py` | Add `TestInvocationContextOnPayload` test class | - ---- - -## Task 1: Create `src/orcapod/invocation.py` leaf module - -**Files:** -- Create: `src/orcapod/invocation.py` - -This task extracts the three invocation-identity items from `side_effects.py` into a new module with no orcapod-internal runtime imports. No behavioral change — only relocation. - -- [ ] **Step 1: Create `src/orcapod/invocation.py`** - -```python -# src/orcapod/invocation.py -"""Invocation identity types for orcapod pipeline elements. - -Provides ``InvocationHashConfig``, ``InvocationContext``, and the internal -``_serialize_component`` helper. These types describe how a single pod -invocation is identified — independent of whether the pod is a side-effect -pod or a function pod. Extracted here so both ``hooks.py`` and -``side_effects.py`` can import them without circular-import risk. -""" - -from __future__ import annotations - -import base64 -import dataclasses -from typing import TYPE_CHECKING, Literal - -if TYPE_CHECKING: - from orcapod.types import ContentHash - - -# --------------------------------------------------------------------------- -# InvocationHashConfig -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass(frozen=True) -class InvocationHashConfig: - """Controls how ``InvocationContext.invocation_hash`` is serialized. - - Args: - encoding: Output encoding — ``"hex"`` (default) or ``"base64"``. - component_length: Bytes of raw digest to use per component. ``None`` - means full digest length. Applied identically to every - ``::``-separated component. - """ - - encoding: Literal["hex", "base64"] = "hex" - component_length: int | None = None - - -def _serialize_component(content_hash: ContentHash, config: InvocationHashConfig) -> str: - """Serialize one ``ContentHash`` component per ``InvocationHashConfig``. - - The method name is always included as a prefix (e.g. ``"arrow_v2.1:abcd1234"``). - Only the digest bytes are subject to truncation via ``component_length``. - - Args: - content_hash: The hash to serialize. - config: Encoding and truncation config. - - Returns: - A string of the form ``"{method}:{encoded_digest}"`` where the digest - is optionally truncated then encoded as hex or base64. - """ - raw = content_hash.digest - if config.component_length is not None: - raw = raw[:config.component_length] - if config.encoding == "base64": - encoded = base64.b64encode(raw).decode("ascii") - else: - encoded = raw.hex() - return f"{content_hash.method}:{encoded}" - - -# --------------------------------------------------------------------------- -# InvocationContext -# --------------------------------------------------------------------------- - - -class InvocationContext: - """Per-invocation context describing a single pod call. - - Carries a deterministic ``invocation_hash`` and metadata about the - current delivery. ``invocation_hash`` is a computed property that - delegates to ``format_id()`` with the pod's default - ``InvocationHashConfig``. ``format_id()`` can be called with a custom - config to re-serialize without recomputation. - - Public fields are read-only by convention (no public setters). - - Available on: - - Side-effect pod functions (injected as ``ctx`` argument). - - Function pod post-run hooks (via ``PostRunPayload.invocation_context``). - - Args: - pod_name: ``pod.label`` of the invoking pod. - pipeline_run_id: The current pipeline run identifier, or ``None`` - for standalone / lazy pipelines. - """ - - def __init__( - self, - pod_name: str, - pipeline_run_id: str | None, - _pipeline_hash_ch: ContentHash, - _record_id_hash_ch: ContentHash, - _hash_config: InvocationHashConfig, - _track_completion: bool, - ) -> None: - self.pod_name = pod_name - self.pipeline_run_id = pipeline_run_id - self._pipeline_hash_ch = _pipeline_hash_ch - self._record_id_hash_ch = _record_id_hash_ch - self._hash_config = _hash_config - self._track_completion = _track_completion - - @property - def invocation_hash(self) -> str: - """Serialized invocation hash — delegates to ``format_id()``.""" - return self.format_id() - - def format_id(self, config: InvocationHashConfig | None = None) -> str: - """Return the invocation hash string with an optional format override. - - Serializes the stored ``ContentHash`` components. Uses ``config`` - if supplied, otherwise the pod's own ``InvocationHashConfig``. - - Args: - config: Optional encoding/truncation override. - - Returns: - A string of the form ``"{component1}::{component2}"`` - (two components when ``track_completion=True``) or - ``"{c1}::{c2}::{run_id}"`` (three components when - ``track_completion=False`` and ``pipeline_run_id`` is not ``None``). - Each component is ``"{method}:{encoded_digest}"``. - """ - cfg = config or self._hash_config - c1 = _serialize_component(self._pipeline_hash_ch, cfg) - c2 = _serialize_component(self._record_id_hash_ch, cfg) - if not self._track_completion and self.pipeline_run_id is not None: - return f"{c1}::{c2}::{self.pipeline_run_id}" - return f"{c1}::{c2}" -``` - -- [ ] **Step 2: Update `side_effects.py` — remove moved definitions, import from `invocation.py`** - -In `src/orcapod/side_effects.py`, replace the three blocks (lines 50–185, which define `InvocationHashConfig`, `_serialize_component`, and `InvocationContext`) with a single import. The replacement goes right after the `_SIDE_EFFECT_RECOMPUTATION_INDEX_COL` constant definition. - -Remove these sections entirely from `side_effects.py`: -- The `InvocationHashConfig` dataclass (lines ~51–63) -- The `_serialize_component` function (lines ~65–86) -- The `InvocationContext` class (lines ~126–185) -- Their section comment headers (`# InvocationHashConfig`, `# InvocationContext`) - -Add this import block after the `_SIDE_EFFECT_RECOMPUTATION_INDEX_COL` line: - -```python -from orcapod.invocation import ( - InvocationContext, - InvocationHashConfig, - _serialize_component, -) -``` - -- [ ] **Step 3: Update `function_pod.py` — switch local import to `invocation.py`** - -In `src/orcapod/core/function_pod.py`, find the local import block inside `_build_invocation_context` (around line 185–189): - -```python - from orcapod.side_effects import ( - InvocationContext, - InvocationHashConfig, - _SIDE_EFFECT_RECOMPUTATION_INDEX_COL, - ) -``` - -Replace with: - -```python - from orcapod.invocation import InvocationContext, InvocationHashConfig - from orcapod.side_effects import _SIDE_EFFECT_RECOMPUTATION_INDEX_COL -``` - -- [ ] **Step 4: Update `__init__.py` — import from `.invocation` instead of `.side_effects`** - -In `src/orcapod/__init__.py`, change: - -```python -from .side_effects import ( - InvocationContext, - InvocationHashConfig, - SideEffectPod, - SideEffectPodConfig, - side_effect_pod, - sink_pod, - tap_pod, -) -``` - -To: - -```python -from .invocation import ( - InvocationContext, - InvocationHashConfig, -) -from .side_effects import ( - SideEffectPod, - SideEffectPodConfig, - side_effect_pod, - sink_pod, - tap_pod, -) -``` - -- [ ] **Step 5: Run existing tests to verify nothing broke** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all existing tests pass. If any fail, they indicate a missed import update — fix before proceeding. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/invocation.py src/orcapod/side_effects.py src/orcapod/core/function_pod.py src/orcapod/__init__.py -git commit -m "refactor(invocation): extract InvocationContext/InvocationHashConfig to leaf module (ITL-531)" -``` - ---- - -## Task 2: Write failing tests for `PostRunPayload.invocation_context` - -**Files:** -- Modify: `tests/test_core/function_pod/test_post_run_hooks.py` - -- [ ] **Step 1: Add `TestInvocationContextOnPayload` test class** - -Append this class to the end of `tests/test_core/function_pod/test_post_run_hooks.py`. Add `from orcapod import InvocationHashConfig` to the existing imports at the top of the file. - -```python -# --------------------------------------------------------------------------- -# InvocationContext on PostRunPayload (ITL-531) -# --------------------------------------------------------------------------- - - -class TestInvocationContextOnPayload: - def test_invocation_context_always_present(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - assert payloads[0].invocation_context is not None - - def test_invocation_hash_is_nonempty_string(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - ctx = payloads[0].invocation_context - assert isinstance(ctx.invocation_hash, str) - assert len(ctx.invocation_hash) > 0 - - def test_format_id_matches_invocation_hash(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - ctx = payloads[0].invocation_context - assert ctx.format_id() == ctx.invocation_hash - - def test_format_id_base64_differs_from_hex(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - ctx = payloads[0].invocation_context - hex_id = ctx.invocation_hash - b64_id = ctx.format_id(InvocationHashConfig(encoding="base64")) - assert hex_id != b64_id - - def test_pipeline_run_id_is_none_standalone(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - stream = pod.process(_make_stream(n=1)) - list(stream.iter_data()) - - assert payloads[0].invocation_context.pipeline_run_id is None - - def test_invocation_hash_deterministic(self): - pod = _make_double_pod() - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(payloads.append) - - # Process the same input stream twice - stream1 = pod.process(_make_stream(n=1)) - list(stream1.iter_data()) - stream2 = pod.process(_make_stream(n=1)) - list(stream2.iter_data()) - - assert len(payloads) == 2 - assert ( - payloads[0].invocation_context.invocation_hash - == payloads[1].invocation_context.invocation_hash - ) - - def test_error_payload_has_invocation_context(self): - def explodes(x: int) -> int: - raise RuntimeError("boom") - - pf = PythonDataFunction(explodes, output_keys="result") - pod = FunctionPod(pf) - payloads: list[PostRunPayload] = [] - pod.add_post_run_hook(HookConfig(fn=payloads.append, on_error="log")) - - stream = pod.process(_make_stream(n=1)) - with pytest.raises(RuntimeError, match="boom"): - list(stream.iter_data()) - - assert len(payloads) == 1 - assert payloads[0].invocation_context is not None - assert isinstance(payloads[0].invocation_context.invocation_hash, str) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload -v -``` - -Expected: All 7 tests FAIL. The first failure should be `AttributeError: 'PostRunPayload' object has no attribute 'invocation_context'` (or similar). If any pass unexpectedly, investigate before proceeding. - ---- - -## Task 3: Add `invocation_context` field to `PostRunPayload` - -**Files:** -- Modify: `src/orcapod/hooks.py` - -- [ ] **Step 1: Add import and field to `PostRunPayload`** - -In `src/orcapod/hooks.py`, add the import after the existing imports: - -```python -from __future__ import annotations - -import dataclasses -from collections.abc import Callable -from datetime import datetime -from enum import Enum -from typing import TYPE_CHECKING, Literal - -from orcapod.invocation import InvocationContext - -if TYPE_CHECKING: - from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -``` - -Then update `PostRunPayload` to add the new field. The field goes after `pod` with a default of `None` so existing positional and keyword construction still works: - -```python -@dataclasses.dataclass(frozen=True) -class PostRunPayload: - """Payload passed to every post-run hook after a pod invocation. - - Attributes: - record_id_hash: String form of ``output.datagram_uuid`` — the same UUID - used as the primary key when the result is stored in a backing database. - ``None`` when ``output`` is ``None`` (filtered row or error). - tag: The input tag for this invocation. Treat as read-only. - input: The input data for this invocation. Treat as read-only. - output: The output data; ``None`` if the function filtered the row out or raised. - stats: Timing and status bundle. - pod: Identity of the pod that produced this result. - invocation_context: Deterministic invocation identity for this row. - Carries ``invocation_hash`` (input-keyed, encoding-configurable), - ``format_id(config)`` for custom serialization, and - ``pipeline_run_id``. ``None`` only when ``PostRunPayload`` is - constructed directly without providing this argument (e.g. in - tests); always populated when built by the pod itself. - """ - - record_id_hash: str | None - tag: TagProtocol - input: DataProtocol - output: DataProtocol | None - stats: RunStats - pod: PodContext - invocation_context: InvocationContext | None = None -``` - -- [ ] **Step 2: Run the new tests — expect partial failure** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload -v -``` - -Expected: Tests fail because `invocation_context` is always `None` (not yet populated by the pod). The assertion `assert payloads[0].invocation_context is not None` should now be the failure point rather than an `AttributeError`. - -- [ ] **Step 3: Run the full existing test suite to confirm no regressions** - -```bash -uv run pytest tests/ -x -q --ignore=tests/test_core/function_pod/test_post_run_hooks.py -``` - -Expected: All pass. `PostRunPayload`'s `= None` default means no existing construction sites break. - ---- - -## Task 4: Thread `run_id` into `_build_post_run_payload` and populate `invocation_context` - -**Files:** -- Modify: `src/orcapod/core/function_pod.py` - -- [ ] **Step 1: Update `_build_post_run_payload` signature and body** - -Find `_build_post_run_payload` in `_FunctionPodBase` (around line 352). Replace the entire method: - -```python - def _build_post_run_payload( - self, - tag: TagProtocol, - data: DataProtocol, - output_data: DataProtocol | None, - started_at: datetime, - finished_at: datetime, - status: InvocationStatus, - exc: Exception | None, - run_id: str | None = None, - ) -> PostRunPayload: - """Build a ``PostRunPayload`` from invocation results. - - Args: - tag: The input tag. - data: The input data. - output_data: The output data, or ``None`` if filtered or errored. - started_at: UTC timestamp when the invocation started. - finished_at: UTC timestamp when compute-or-lookup completed. - status: Invocation status (``COMPUTED``, ``HIT``, or ``ERROR``). - exc: The exception raised, if ``status == ERROR``; ``None`` otherwise. - run_id: Pipeline run identifier, or ``None`` in standalone mode. - Forwarded to ``InvocationContext.pipeline_run_id``. - - Returns: - A ``PostRunPayload`` ready to pass to registered hooks. - """ - record_id = ( - str(output_data.datagram_uuid) if output_data is not None else None - ) - invocation_context = self._build_invocation_context(tag, data, run_id=run_id) - return PostRunPayload( - record_id_hash=record_id, - tag=tag, - input=data, - output=output_data, - stats=RunStats( - duration_ms=(finished_at - started_at).total_seconds() * 1000, - status=status, - started_at=started_at, - finished_at=finished_at, - error=exc, - ), - pod=PodContext( - label=self.label, - pod_hash=self.content_hash().to_string(), - ), - invocation_context=invocation_context, - ) -``` - -- [ ] **Step 2: Update `_invoke_with_hooks` to pass `run_id` to `_build_post_run_payload`** - -Find `_invoke_with_hooks` in `_FunctionPodBase` (around line 397). There are two calls to `_build_post_run_payload` inside it — the error path and the success path. Update both to pass `run_id=run_id`: - -```python - def _invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - """Call ``process_data``, time it, and fire post-run hooks. - - When ``_post_run_hooks`` is empty, delegates directly to - ``process_data`` with zero overhead. Override in subclasses (e.g. - ``CachedFunctionPod``) to supply a different ``InvocationStatus``. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional data execution logger forwarded to ``process_data``. - run_id: Pipeline run identifier forwarded to ``process_data`` and - ``PostRunPayload.invocation_context``. - - Returns: - A ``(tag, output_data)`` tuple. - """ - if not self._post_run_hooks: - return self.process_data(tag, data, logger=logger, run_id=run_id) - - started_at = datetime.now(timezone.utc) - out_tag = tag - output_data: DataProtocol | None = None - - try: - out_tag, output_data = self.process_data(tag, data, logger=logger, run_id=run_id) - except Exception as exc: - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, None, started_at, finished_at, - InvocationStatus.ERROR, exc, run_id=run_id, - ) - ) - raise # bare raise — preserves the original traceback exactly - - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, output_data, started_at, finished_at, - InvocationStatus.COMPUTED, None, run_id=run_id, - ) - ) - return out_tag, output_data -``` - -- [ ] **Step 3: Update `_async_invoke_with_hooks` to pass `run_id` to `_build_post_run_payload`** - -Find `_async_invoke_with_hooks` in `_FunctionPodBase` (around line 448). Update both `_build_post_run_payload` calls inside it to pass `run_id=run_id`: - -```python - async def _async_invoke_with_hooks( - self, - tag: TagProtocol, - data: DataProtocol, - *, - logger: DataExecutionLoggerProtocol | None = None, - run_id: str | None = None, - ) -> tuple[TagProtocol, DataProtocol | None]: - """Async counterpart of ``_invoke_with_hooks``. - - When ``_post_run_hooks`` is empty, delegates directly to - ``async_process_data`` with zero overhead. - - Args: - tag: The tag associated with the data. - data: The input data to process. - logger: Optional data execution logger forwarded to - ``async_process_data``. - run_id: Pipeline run identifier forwarded to ``async_process_data`` - and ``PostRunPayload.invocation_context``. - - Returns: - A ``(tag, output_data)`` tuple. - """ - if not self._post_run_hooks: - return await self.async_process_data(tag, data, logger=logger, run_id=run_id) - - started_at = datetime.now(timezone.utc) - out_tag = tag - output_data: DataProtocol | None = None - - try: - out_tag, output_data = await self.async_process_data( - tag, data, logger=logger, run_id=run_id - ) - except Exception as exc: - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, None, started_at, finished_at, - InvocationStatus.ERROR, exc, run_id=run_id, - ) - ) - raise # bare raise — preserves the original traceback exactly - - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, output_data, started_at, finished_at, - InvocationStatus.COMPUTED, None, run_id=run_id, - ) - ) - return out_tag, output_data -``` - -- [ ] **Step 4: Run the new test class — all 7 tests should now pass** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload -v -``` - -Expected output: -``` -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_invocation_context_always_present PASSED -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_invocation_hash_is_nonempty_string PASSED -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_format_id_matches_invocation_hash PASSED -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_format_id_base64_differs_from_hex PASSED -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_pipeline_run_id_is_none_standalone PASSED -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_invocation_hash_deterministic PASSED -tests/test_core/function_pod/test_post_run_hooks.py::TestInvocationContextOnPayload::test_error_payload_has_invocation_context PASSED - -7 passed -``` - -- [ ] **Step 5: Update `CachedFunctionPod._invoke_with_hooks` to pass `run_id`** - -`CachedFunctionPod` (in `src/orcapod/core/cached_function_pod.py`) overrides -`_invoke_with_hooks` and `_async_invoke_with_hooks`, calling `_build_post_run_payload` -without `run_id`. With the new `run_id` keyword argument added in Step 1, the code -still compiles (default `None`), but `pipeline_run_id` would silently be `None` -even when a run_id is in scope. Fix both calls in each override. - -In `CachedFunctionPod._invoke_with_hooks`, update the two `_build_post_run_payload` calls: - -```python - except Exception as exc: - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, None, started_at, finished_at, - InvocationStatus.ERROR, exc, run_id=run_id, - ) - ) - raise # bare raise — preserves the original traceback exactly - - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, output_data, started_at, finished_at, status, None, - run_id=run_id, - ) - ) -``` - -In `CachedFunctionPod._async_invoke_with_hooks`, make the same two changes: - -```python - except Exception as exc: - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, None, started_at, finished_at, - InvocationStatus.ERROR, exc, run_id=run_id, - ) - ) - raise # bare raise — preserves the original traceback exactly - - finished_at = datetime.now(timezone.utc) - self._fire_post_run_hooks( - self._build_post_run_payload( - tag, data, output_data, started_at, finished_at, status, None, - run_id=run_id, - ) - ) -``` - -- [ ] **Step 6: Run the full test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: All tests pass. Fix any failures before committing. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/hooks.py src/orcapod/core/function_pod.py src/orcapod/core/cached_function_pod.py tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "feat(hooks): add invocation_context to PostRunPayload (ITL-531)" -``` - ---- - -## Task 5: Export `InvocationContext` from public API and finalize - -**Files:** -- Modify: `src/orcapod/__init__.py` (verify — already done in Task 1) -- Modify: `tests/test_core/function_pod/test_post_run_hooks.py` (add public API import check) - -- [ ] **Step 1: Verify `InvocationContext` is still exported from `orcapod`** - -```bash -uv run python -c "from orcapod import InvocationContext, InvocationHashConfig; print('OK')" -``` - -Expected: `OK` - -- [ ] **Step 2: Add public API import assertion to the existing `test_public_api_imports` test** - -In `tests/test_core/function_pod/test_post_run_hooks.py`, find `test_public_api_imports` in `TestDecoratorConvenience` and add two assertions: - -```python - def test_public_api_imports(self): - import orcapod - assert hasattr(orcapod, "PostRunPayload") - assert hasattr(orcapod, "HookConfig") - assert hasattr(orcapod, "InvocationStatus") - assert hasattr(orcapod, "RunStats") - assert hasattr(orcapod, "PodContext") - assert hasattr(orcapod, "InvocationContext") # ITL-531 - assert hasattr(orcapod, "InvocationHashConfig") # ITL-531 -``` - -- [ ] **Step 3: Run the updated test** - -```bash -uv run pytest tests/test_core/function_pod/test_post_run_hooks.py::TestDecoratorConvenience::test_public_api_imports -v -``` - -Expected: PASS - -- [ ] **Step 4: Run the complete test suite one final time** - -```bash -uv run pytest tests/ -q -``` - -Expected: All tests pass, no warnings about import paths. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_core/function_pod/test_post_run_hooks.py -git commit -m "test(hooks): verify InvocationContext/InvocationHashConfig in public API (ITL-531)" -``` - ---- - -## Task 6: Push branch and open PR - -- [ ] **Step 1: Push branch** - -```bash -git push -u origin eywalker/itl-531-align-postrunpayload-with-sideeffectpod-invocationcontext -``` - -- [ ] **Step 2: Open PR against `main`** - -```bash -gh pr create \ - --title "feat(hooks): align PostRunPayload with InvocationContext (ITL-531)" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -- Extracts `InvocationContext` and `InvocationHashConfig` out of `side_effects.py` into a new stdlib-only leaf module `src/orcapod/invocation.py` -- Adds `invocation_context: InvocationContext | None = None` to `PostRunPayload` — populated on every payload built by the pod -- Threads `run_id` through `_build_post_run_payload` so `pipeline_run_id` is accessible via the context - -Closes ITL-531 - -## Test plan - -- [ ] All existing tests pass unmodified (additive `= None` default) -- [ ] `TestInvocationContextOnPayload` (7 new tests) all pass -- [ ] `test_public_api_imports` verifies `InvocationContext` and `InvocationHashConfig` remain in the public API - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` diff --git a/superpowers/plans/2026-07-22-empty-table-nullability.md b/superpowers/plans/2026-07-22-empty-table-nullability.md deleted file mode 100644 index 79ee505bf..000000000 --- a/superpowers/plans/2026-07-22-empty-table-nullability.md +++ /dev/null @@ -1,730 +0,0 @@ -# Empty Table Nullability Fix Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix five empty-table construction sites that silently drop field nullability, causing `error_policy="continue"` pipelines to abort when a failing function feeds a `Join`. - -**Architecture:** Add a single `make_empty_table(python_schema, type_converter)` utility to `arrow_utils.py` that uses the existing `python_schema_to_arrow_schema` method (which correctly preserves `nullable=False` for required fields). Replace all five buggy call sites with this utility. Write tests first at every step. - -**Tech Stack:** Python, PyArrow, pytest, `uv run` - ---- - -## File Map - -| Action | Path | Purpose | -|---|---|---| -| Modify | `src/orcapod/utils/arrow_utils.py` | Add `make_empty_table()` | -| Modify | `src/orcapod/core/nodes/operator_node.py` | Fix `_make_empty_table()` body | -| Modify | `src/orcapod/pipeline/sync_orchestrator.py` | Fix `_materialize_as_stream()` empty branch | -| Modify | `src/orcapod/side_effects.py` | Fix `SideEffectFunctionStream.as_table()` and `SideEffectJobFunctionStream.as_table()` | -| Modify | `src/orcapod/core/sources/derived_source.py` | Fix `DerivedSource._get_stream()` empty branch | -| Modify | `tests/test_utils/test_arrow_utils.py` | Add `make_empty_table` unit tests | -| Create | `tests/test_pipeline/test_error_policy_continue.py` | Integration test for the bug scenario | -| Modify | `tests/test_core/nodes/test_function_node_iteration.py` | Strengthen nullability assertion | -| Modify | `tests/test_core/sources/test_derived_source.py` | Add nullability regression test | -| Modify | `tests/test_core/side_effect_function/test_side_effect_function_pod.py` | Add nullability regression test | -| Modify | `DESIGN_ISSUES.md` | Mark CC1 resolved | - ---- - -## Task 1: Create the feature branch - -**Files:** none (git only) - -- [ ] **Step 1: Check out the feature branch** - -```bash -cd /path/to/orcapod-python # your actual working directory -git checkout -b eywalker/itl-563-empty-failed-outputs-lose-tag-nullability-and-abort -git branch --show-current -``` - -Expected output: `eywalker/itl-563-empty-failed-outputs-lose-tag-nullability-and-abort` - ---- - -## Task 2: Unit tests for `make_empty_table` (write tests first, implementation not yet added) - -**Files:** -- Modify: `tests/test_utils/test_arrow_utils.py` - -- [ ] **Step 1: Write four failing tests at the bottom of the existing test file** - -Append to `tests/test_utils/test_arrow_utils.py`: - -```python -# --------------------------------------------------------------------------- -# make_empty_table — ITL-563 -# --------------------------------------------------------------------------- - - -class TestMakeEmptyTable: - """make_empty_table preserves field nullability from python_schema.""" - - def _converter(self): - from orcapod.contexts import DataContext - return DataContext().type_converter - - def test_required_fields_are_non_nullable(self): - """Plain types produce nullable=False Arrow fields.""" - from orcapod.utils.arrow_utils import make_empty_table - - schema = {"name": str, "count": int} - table = make_empty_table(schema, self._converter()) - - assert table.num_rows == 0 - assert table.schema.field("name").nullable is False - assert table.schema.field("count").nullable is False - - def test_optional_fields_are_nullable(self): - """Optional types produce nullable=True Arrow fields.""" - from orcapod.utils.arrow_utils import make_empty_table - - schema = {"name": str | None, "count": int | None} - table = make_empty_table(schema, self._converter()) - - assert table.num_rows == 0 - assert table.schema.field("name").nullable is True - assert table.schema.field("count").nullable is True - - def test_mixed_nullability_per_field(self): - """Mixed schema: required field non-nullable, optional field nullable.""" - from orcapod.utils.arrow_utils import make_empty_table - - schema = {"subject": str, "score": int | None} - table = make_empty_table(schema, self._converter()) - - assert table.num_rows == 0 - assert table.schema.field("subject").nullable is False - assert table.schema.field("score").nullable is True - - def test_round_trips_through_arrow_table_stream(self): - """Python schema → make_empty_table → ArrowTableStream.output_schema() is identity.""" - from orcapod.utils.arrow_utils import make_empty_table - from orcapod.core.streams import ArrowTableStream - - converter = self._converter() - python_schema = {"subject": str, "score": int | None} - table = make_empty_table(python_schema, converter) - - # ArrowTableStream requires at least one data column; tag=subject, data=score - stream = ArrowTableStream(table, tag_columns=["subject"]) - _, data_schema = stream.output_schema() - - assert data_schema["score"] == (int | None) -``` - -- [ ] **Step 2: Run tests to verify they fail (function not yet defined)** - -```bash -uv run pytest tests/test_utils/test_arrow_utils.py::TestMakeEmptyTable -v -``` - -Expected: 4 failures with `ImportError` or `cannot import name 'make_empty_table'`. - ---- - -## Task 3: Implement `make_empty_table` in `arrow_utils.py` - -**Files:** -- Modify: `src/orcapod/utils/arrow_utils.py` - -- [ ] **Step 1: Add the function after the last import block, before `schema_select`** - -In `src/orcapod/utils/arrow_utils.py`, insert after line 14 (the `pa = LazyModule("pyarrow")` line) and before `def schema_select`: - -```python - -def make_empty_table(python_schema: "Mapping[str, Any]", type_converter: Any) -> "pa.Table": - """Return a zero-row PyArrow table whose field nullability matches ``python_schema``. - - Uses ``python_schema_to_arrow_schema`` so that plain types (``str``, ``int``, …) - produce ``nullable=False`` fields and Optional types (``str | None``) produce - ``nullable=True`` fields. This preserves the bidirectional round-trip through - ``ArrowTableStream.output_schema()``. - - Args: - python_schema: Mapping of field name to Python type annotation. - type_converter: A ``UniversalTypeConverter`` instance. - - Returns: - A zero-row ``pa.Table`` with the correct Arrow schema. - """ - arrow_schema = type_converter.python_schema_to_arrow_schema(python_schema) - return pa.Table.from_batches([], schema=arrow_schema) -``` - -Also update the `Mapping` import at the top — it is already imported from `collections.abc`. - -- [ ] **Step 2: Run unit tests to verify they pass** - -```bash -uv run pytest tests/test_utils/test_arrow_utils.py::TestMakeEmptyTable -v -``` - -Expected: 4 tests PASS. - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/utils/arrow_utils.py tests/test_utils/test_arrow_utils.py -git commit -m "feat(arrow_utils): add make_empty_table preserving field nullability (ITL-563)" -``` - ---- - -## Task 4: Fix `operator_node.py` — `_make_empty_table()` - -**Files:** -- Modify: `src/orcapod/core/nodes/operator_node.py` -- Modify: `tests/test_core/nodes/test_function_node_iteration.py` - -`operator_node.py` already has a `_make_empty_table()` method (lines 835–849) called from -three internal sites. We replace its body with a call to `arrow_utils.make_empty_table()`. - -- [ ] **Step 1: Strengthen the existing test to assert nullability (it currently only checks column names)** - -In `tests/test_core/nodes/test_function_node_iteration.py`, update `test_as_table_empty_schema_matches_non_empty_schema`: - -```python -def test_as_table_empty_schema_matches_non_empty_schema(): - """as_table() empty table has the same columns and nullability as the populated table.""" - db = InMemoryArrowDatabase() - node_after = _make_node(db=db) - node_after.run() - full_table = node_after.as_table() - - node_before = _make_node() - empty_table = node_before.as_table() - - assert empty_table.num_rows == 0 - assert full_table.num_rows > 0 - assert set(empty_table.column_names) == set(full_table.column_names) - # Nullability must match — this failed before the fix - assert all( - empty_table.schema.field(n).nullable == full_table.schema.field(n).nullable - for n in empty_table.column_names - ) -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_iteration.py::test_as_table_empty_schema_matches_non_empty_schema -v -``` - -Expected: FAIL — `nullable` flags differ for required fields. - -- [ ] **Step 3: Fix the body of `_make_empty_table()` in `operator_node.py`** - -Replace the current body of `_make_empty_table()` (lines 835–849) with: - -```python - def _make_empty_table(self) -> "pa.Table": - """Build a zero-row PyArrow table matching this node's full output schema. - - Uses ``arrow_utils.make_empty_table`` so field nullability is preserved - (``nullable=False`` for required fields, ``nullable=True`` for optional). - Requires ``self._operator is not None`` (pre-existing limitation shared - with ``_replay_from_cache``). - """ - from orcapod.utils.arrow_utils import make_empty_table - - tag_schema, data_schema = self.output_schema() - return make_empty_table( - {**tag_schema, **data_schema}, - self.data_context.type_converter, - ) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -uv run pytest tests/test_core/nodes/test_function_node_iteration.py::test_as_table_empty_schema_matches_non_empty_schema -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run the full `test_core/nodes` suite to check for regressions** - -```bash -uv run pytest tests/test_core/nodes/ -v -``` - -Expected: all PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/nodes/operator_node.py tests/test_core/nodes/test_function_node_iteration.py -git commit -m "fix(operator_node): use make_empty_table to preserve field nullability (ITL-563)" -``` - ---- - -## Task 5: Fix `sync_orchestrator.py` — `_materialize_as_stream()` + integration test - -**Files:** -- Create: `tests/test_pipeline/test_error_policy_continue.py` -- Modify: `src/orcapod/pipeline/sync_orchestrator.py` - -- [ ] **Step 1: Write the failing integration tests** - -Create `tests/test_pipeline/test_error_policy_continue.py`: - -```python -"""Integration tests for error_policy='continue' + Join schema compatibility. - -Regression for ITL-563: failed functions produced empty tables with wrong -nullable flags, causing Join to raise InputValidationError and abort the pipeline. -""" -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.operators.join import Join -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase -from orcapod.pipeline.job import PipelineJob - - -def _make_source_with_required_field(subjects: list[str], values: list[int]) -> ArrowTableSource: - """Source with tag=subject (str, non-nullable) and data=value (int, non-nullable).""" - schema = pa.schema([ - pa.field("subject", pa.large_string(), nullable=False), - pa.field("value", pa.int64(), nullable=False), - ]) - table = pa.table( - {"subject": subjects, "value": values}, - schema=schema, - ) - return ArrowTableSource(table, tag_columns=["subject"]) - - -def test_failed_function_with_join_does_not_abort_pipeline(): - """Topology: source → failing_function → Join ← source. - - Under error_policy='continue', the failing function should be logged, - the Join should produce zero rows (empty × non-empty = empty), and - orchestration should complete without raising. - """ - src = _make_source_with_required_field(["a", "b"], [1, 2]) - - def always_fails(value: int) -> int: - raise RuntimeError("intentional failure") - - pf = PythonDataFunction(always_fails, output_keys="transformed") - failing_pod = FunctionPod(pf) - - job = PipelineJob(name="test_join_continue", store=InMemoryArrowDatabase()) - with job: - failing_out = failing_pod(src, label="failing") - Join()(failing_out, src, label="joined") - - # Should complete without raising despite the failing function - job.run(error_policy="continue") - - joined_records = job.nodes["joined"].get_all_records() - # Join of empty × non-empty = empty (zero rows), not an error - assert joined_records is None or joined_records.num_rows == 0 - - -def test_empty_buffer_schema_preserves_nullability(): - """_materialize_as_stream on an empty buffer preserves required field nullability.""" - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - from orcapod.core.nodes.function_node import FunctionJobNode - - src = _make_source_with_required_field(["x"], [10]) - - def identity(value: int) -> int: - return value - - pf = PythonDataFunction(identity, output_keys="result") - pod = FunctionPod(pf) - - db = InMemoryArrowDatabase() - job = PipelineJob(name="test_empty_schema", store=db) - with job: - pod(src, label="fn") - - # Don't run — buffer will be empty - node = job.nodes["fn"] - stream = SyncPipelineOrchestrator._materialize_as_stream([], node) - tag_schema, data_schema = stream.output_schema() - - # "result" is declared as int (required), must NOT become int | None - assert data_schema["result"] == int -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -```bash -uv run pytest tests/test_pipeline/test_error_policy_continue.py -v -``` - -Expected: `test_failed_function_with_join_does_not_abort_pipeline` raises `InputValidationError`; `test_empty_buffer_schema_preserves_nullability` asserts `int | None` but expects `int`. - -- [ ] **Step 3: Fix `_materialize_as_stream()` in `sync_orchestrator.py`** - -In `src/orcapod/pipeline/sync_orchestrator.py`, replace the empty-branch of -`_materialize_as_stream()` (lines 181–198). The current code is: - -```python - if not buf: - # Build an empty stream with the correct schema from the upstream node - tag_schema, data_schema = upstream_node.output_schema( - columns={"system_tags": True, "source": True} - ) - type_converter = upstream_node.data_context.type_converter - empty_fields = {} - for name, py_type in {**tag_schema, **data_schema}.items(): - arrow_type = type_converter.python_type_to_arrow_type(py_type) - empty_fields[name] = pa.array([], type=arrow_type) - empty_table = pa.table(empty_fields) - tag_keys = upstream_node.keys()[0] - return ArrowTableStream( - empty_table, - tag_columns=tag_keys, - producer=upstream_node.producer, - upstreams=upstream_node.upstreams, - ) -``` - -Replace with: - -```python - if not buf: - # Build an empty stream preserving declared field nullability. - tag_schema, data_schema = upstream_node.output_schema( - columns={"system_tags": True, "source": True} - ) - type_converter = upstream_node.data_context.type_converter - empty_table = arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, type_converter - ) - tag_keys = upstream_node.keys()[0] - return ArrowTableStream( - empty_table, - tag_columns=tag_keys, - producer=upstream_node.producer, - upstreams=upstream_node.upstreams, - ) -``` - -Note: `arrow_utils` is already imported at line 176 (`from orcapod.utils import arrow_utils`). - -- [ ] **Step 4: Run the integration tests to verify they pass** - -```bash -uv run pytest tests/test_pipeline/test_error_policy_continue.py -v -``` - -Expected: both PASS. - -- [ ] **Step 5: Run the full pipeline test suite to check for regressions** - -```bash -uv run pytest tests/test_pipeline/ -v -``` - -Expected: all PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/sync_orchestrator.py tests/test_pipeline/test_error_policy_continue.py -git commit -m "fix(sync_orchestrator): use make_empty_table to preserve field nullability (ITL-563)" -``` - ---- - -## Task 6: Fix `side_effects.py` — two `as_table()` methods - -**Files:** -- Modify: `src/orcapod/side_effects.py` -- Modify: `tests/test_core/side_effect_function/test_side_effect_function_pod.py` - -- [ ] **Step 1: Write a failing test for `SideEffectFunctionStream.as_table()`** - -Append to `tests/test_core/side_effect_function/test_side_effect_function_pod.py`: - -```python -class TestSideEffectEmptyTableNullability: - """Regression for ITL-563: empty as_table() must preserve field nullability.""" - - def test_empty_stream_schema_preserves_required_fields(self): - """SideEffectFunctionStream.as_table() with no rows preserves nullable=False.""" - from orcapod.core.function_pod import FunctionPod - from orcapod.side_effects import InvocationContext - - def my_fn(value: int, ctx: InvocationContext) -> str: - return str(value) - - pod = FunctionPod.from_fn(my_fn, output_keys=["result"], ctx_arg_name="ctx") - stream = _make_stream(n=0) # empty input → empty output - - # Build an empty stream (iter_data yields nothing) - from orcapod.side_effects import SideEffectFunctionStream - se_stream = SideEffectFunctionStream(pod=pod, input_stream=stream) - - table = se_stream.as_table() - assert table.num_rows == 0 - # "result" is declared as str (required) — must not become str | None - assert table.schema.field("result").nullable is False -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -uv run pytest "tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectEmptyTableNullability" -v -``` - -Expected: FAIL — `nullable` is `True` but should be `False`. - -- [ ] **Step 3: Fix `SideEffectFunctionStream.as_table()` in `side_effects.py`** - -Locate the empty-branch inside `SideEffectFunctionStream.as_table()` (around lines 165–176). -Current code: - -```python - if not tag_tables: - # Return an empty table with the correct schema - tag_schema, data_schema = self.output_schema( - columns=column_config - ) - tc = self._pod.data_context.type_converter - fields = {} - for name, py_type in {**tag_schema, **data_schema}.items(): - fields[name] = pa.array( - [], type=tc.python_type_to_arrow_type(py_type) - ) - return pa.table(fields) -``` - -Replace with: - -```python - if not tag_tables: - # Return an empty table preserving declared field nullability. - tag_schema, data_schema = self.output_schema( - columns=column_config - ) - tc = self._pod.data_context.type_converter - return arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, tc - ) -``` - -Note: `arrow_utils` is already imported in the `as_table()` body (`from orcapod.utils import arrow_utils`). - -- [ ] **Step 4: Fix `SideEffectJobFunctionStream.as_table()` in `side_effects.py`** - -Locate the empty-branch inside `SideEffectJobFunctionStream.as_table()` (around lines 728–736). -Current code: - -```python - if not tag_tables: - tag_schema, data_schema = self.output_schema(columns=column_config) - tc = self._pod.data_context.type_converter - fields = {} - for name, py_type in {**tag_schema, **data_schema}.items(): - fields[name] = pa.array( - [], type=tc.python_type_to_arrow_type(py_type) - ) - return pa.table(fields) -``` - -Replace with: - -```python - if not tag_tables: - tag_schema, data_schema = self.output_schema(columns=column_config) - tc = self._pod.data_context.type_converter - return arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, tc - ) -``` - -Note: `arrow_utils` is already imported in this method's body too. - -- [ ] **Step 5: Run the test to verify it passes** - -```bash -uv run pytest "tests/test_core/side_effect_function/test_side_effect_function_pod.py::TestSideEffectEmptyTableNullability" -v -``` - -Expected: PASS. - -- [ ] **Step 6: Run the full side-effect test suite to check for regressions** - -```bash -uv run pytest tests/test_core/side_effect_function/ tests/test_core/side_effect_pod/ -v -``` - -Expected: all PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/orcapod/side_effects.py tests/test_core/side_effect_function/test_side_effect_function_pod.py -git commit -m "fix(side_effects): use make_empty_table to preserve field nullability (ITL-563)" -``` - ---- - -## Task 7: Fix `derived_source.py` — `DerivedSource._get_stream()` - -**Files:** -- Modify: `src/orcapod/core/sources/derived_source.py` -- Modify: `tests/test_core/sources/test_derived_source.py` - -- [ ] **Step 1: Write a failing test for `DerivedSource` empty cache nullability** - -Append to `tests/test_core/sources/test_derived_source.py`: - -```python -def test_derived_source_empty_cache_preserves_nullability(): - """DerivedSource._get_stream() empty table preserves nullable=False for required fields. - - Regression for ITL-563: pa.field() without nullable=False defaulted to True. - """ - # _make_node() builds a FunctionJobNode with data output "result: int" (required). - node = _make_node() - source = node.as_source() # DerivedSource backed by unrun node → empty cache - - # as_table() triggers _get_stream() which hits the empty-cache branch - table = source.as_table() - - assert table.num_rows == 0 - # "result" is int (required), must not become int | None - result_field = table.schema.field("result") - assert result_field.nullable is False -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -uv run pytest tests/test_core/sources/test_derived_source.py::test_derived_source_empty_cache_preserves_nullability -v -``` - -Expected: FAIL — `nullable` is `True` but should be `False`. - -- [ ] **Step 3: Fix `DerivedSource._get_stream()` in `derived_source.py`** - -Locate the `records is None` branch in `_get_stream()` (lines 75–95). Current code: - -```python - if records is None: - # Build empty table with correct schema - tag_schema, data_schema = self._origin.output_schema() - tag_keys = self._origin.keys()[0] - tc = self.data_context.type_converter - fields = [ - pa.field(k, tc.python_type_to_arrow_type(tag_schema[k])) - for k in tag_keys - ] - fields += [ - pa.field(k, tc.python_type_to_arrow_type(v)) - for k, v in data_schema.items() - ] - arrow_schema = pa.schema(fields) - self._cached_table = pa.table( - {f.name: pa.array([], type=f.type) for f in arrow_schema}, - schema=arrow_schema, - ) -``` - -Replace with: - -```python - if records is None: - # Build empty table preserving declared field nullability. - from orcapod.utils.arrow_utils import make_empty_table - - tag_schema, data_schema = self._origin.output_schema() - tag_keys = self._origin.keys()[0] - tc = self.data_context.type_converter - python_schema = {k: tag_schema[k] for k in tag_keys} - python_schema.update(data_schema) - self._cached_table = make_empty_table(python_schema, tc) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -uv run pytest tests/test_core/sources/test_derived_source.py::test_derived_source_empty_cache_preserves_nullability -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run the full sources test suite to check for regressions** - -```bash -uv run pytest tests/test_core/sources/ -v -``` - -Expected: all PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/sources/derived_source.py tests/test_core/sources/test_derived_source.py -git commit -m "fix(derived_source): use make_empty_table to preserve field nullability (ITL-563)" -``` - ---- - -## Task 8: Mark DESIGN_ISSUES.md resolved + run full test suite - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Mark CC1 resolved in `DESIGN_ISSUES.md`** - -Update the CC1 entry: change `**Status:** in progress` to `**Status:** resolved` and append a **Fix:** line: - -``` -**Fix:** Extracted ``arrow_utils.make_empty_table(python_schema, type_converter)`` using -``python_schema_to_arrow_schema`` + ``pa.Table.from_batches([], schema=...)``. Replaced all -five buggy sites. Added unit tests in ``test_arrow_utils.py``, integration test in -``test_error_policy_continue.py``, and per-site regression tests. PR: ITL-563. -``` - -- [ ] **Step 2: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short -``` - -Expected: all PASS (or pre-existing failures only — check against `git stash` baseline if any fail). - -- [ ] **Step 3: Final commit** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs(design-issues): mark CC1 resolved — empty table nullability fix (ITL-563)" -``` - ---- - -## Self-Review Checklist (completed inline) - -| Spec requirement | Covered by task | -|---|---| -| `make_empty_table` in `arrow_utils.py` | Task 3 | -| Fix `sync_orchestrator._materialize_as_stream` | Task 5 | -| Fix `operator_node._make_empty_table` | Task 4 | -| Fix `SideEffectFunctionStream.as_table` | Task 6 | -| Fix `SideEffectJobFunctionStream.as_table` | Task 6 | -| Fix `DerivedSource._get_stream` | Task 7 | -| Unit tests: required / optional / mixed / round-trip | Task 2 | -| Integration test: failing fn + Join + continue policy | Task 5 | -| Integration test: empty buffer schema preserves nullability | Task 5 | -| Regression: side-effect empty table nullability | Task 6 | -| Regression: DerivedSource empty cache nullability | Task 7 | -| Strengthen `test_as_table_empty_schema_matches_non_empty_schema` | Task 4 | -| DESIGN_ISSUES.md updated | Tasks 1 (in progress), 8 (resolved) | diff --git a/superpowers/plans/2026-08-05-itl-603-directory-exclude-by-pattern.md b/superpowers/plans/2026-08-05-itl-603-directory-exclude-by-pattern.md deleted file mode 100644 index 0864644d4..000000000 --- a/superpowers/plans/2026-08-05-itl-603-directory-exclude-by-pattern.md +++ /dev/null @@ -1,535 +0,0 @@ -# Directory exclude-by-pattern: relative path matching (ITL-603) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Change `op.Directory` glob filtering to match against POSIX relative paths (via `PurePosixPath.match()`) instead of bare entry names, update the callable signature accordingly, improve docstrings, and add explicit test coverage for the hash invariants. - -**Architecture:** `_hash_dir` gains a `root` parameter so every recursive call can compute `child.relative_to(root)` as a `PurePosixPath`. `_compile_ignore` switches from `fnmatch.fnmatch(entry.name, pat)` to `PurePosixPath(relative).match(pat)`. The callable type hint in both `directory_hashers.py` and `directory_type.py` changes from `Callable[[UPath], bool]` to `Callable[[PurePosixPath], bool]`. All existing tests continue to pass because `PurePosixPath.name` behaves identically to `UPath.name` for callable filters that only inspect `.name`. - -**Tech Stack:** Python stdlib (`pathlib.PurePosixPath`), `uv run pytest` - -**Spec:** `superpowers/specs/2026-08-05-itl-603-directory-exclude-by-pattern-design.md` - -**Branch:** `eywalker/itl-603-opdirectory-optional-exclude-by-pattern-argument-for-hashing` - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/hashing/directory_hashers.py` | Add `root` param to `_hash_dir`; switch to `PurePosixPath.match()`; update imports and type hints | -| `src/orcapod/extension_types/directory_type.py` | Update `ignore=` type hint and docstring | -| `tests/test_hashing/test_directory_handler.py` | Add 4 new tests in `TestBasicDirectoryHasher` | - ---- - -## Task 0: Create and check out the feature branch - -**Files:** none - -- [ ] **Step 1: Create the branch** - -```bash -cd /path/to/orcapod-python -git checkout -b eywalker/itl-603-opdirectory-optional-exclude-by-pattern-argument-for-hashing -git branch --show-current -``` - -Expected output: `eywalker/itl-603-opdirectory-optional-exclude-by-pattern-argument-for-hashing` - ---- - -## Task 1: Write the failing test that drives the implementation - -The test `test_relative_path_pattern_scopes_to_subdirectory` is the only one that fails with -current code — `fnmatch.fnmatch("app.pyc", "sub/*.pyc")` returns `False` because fnmatch -matches only against the filename, so `"sub/*.pyc"` never excludes anything today. - -**Files:** -- Modify: `tests/test_hashing/test_directory_handler.py` - -- [ ] **Step 1: Add the failing test inside `TestBasicDirectoryHasher`** - -Open `tests/test_hashing/test_directory_handler.py`. Inside `class TestBasicDirectoryHasher`, -add after the last test in that class: - -```python -def test_relative_path_pattern_scopes_to_subdirectory(self, tmp_path): - """Pattern 'sub/*.pyc' must exclude .pyc files in sub/ but not in other/.""" - base = tmp_path / "base" - base.mkdir() - (base / "sub").mkdir() - (base / "other").mkdir() - (base / "sub" / "app.py").write_bytes(b"code") - (base / "sub" / "app.pyc").write_bytes(b"compiled in sub") - (base / "other" / "app.py").write_bytes(b"code") - - # reference: same tree but with sub/app.pyc physically absent - ref = tmp_path / "ref" - ref.mkdir() - (ref / "sub").mkdir() - (ref / "other").mkdir() - (ref / "sub" / "app.py").write_bytes(b"code") - (ref / "other" / "app.py").write_bytes(b"code") - - hasher = BasicDirectoryHasher(file_hasher=FileHasher()) - # "sub/*.pyc" should exclude sub/app.pyc → same hash as ref - assert hasher.hash_directory(base, ignore=["sub/*.pyc"]) == hasher.hash_directory(ref) - # "other/*.pyc" should not match sub/app.pyc → different hash from ref - assert hasher.hash_directory(base, ignore=["other/*.pyc"]) != hasher.hash_directory(ref) -``` - -- [ ] **Step 2: Run it to verify it fails** - -```bash -uv run pytest tests/test_hashing/test_directory_handler.py::TestBasicDirectoryHasher::test_relative_path_pattern_scopes_to_subdirectory -v -``` - -Expected: **FAIL** — `AssertionError` on the first assert (both hashes are equal when they -shouldn't differ yet, because `"sub/*.pyc"` currently matches nothing and acts like no filter). - ---- - -## Task 2: Implement relative-path matching in `directory_hashers.py` - -**Files:** -- Modify: `src/orcapod/hashing/directory_hashers.py` - -- [ ] **Step 1: Update imports** - -At the top of `src/orcapod/hashing/directory_hashers.py`, replace: - -```python -import fnmatch -import hashlib -import logging -import os -from collections.abc import Callable - -from upath import UPath -``` - -with: - -```python -import hashlib -import logging -import os -from collections.abc import Callable -from pathlib import PurePosixPath - -from upath import UPath -``` - -(`fnmatch` is no longer used after this task.) - -- [ ] **Step 2: Update `_compile_ignore`** - -Replace the entire `_compile_ignore` function (lines ~19–41) with: - -```python -def _compile_ignore( - ignore: Callable[[PurePosixPath], bool] | list[str] | None, -) -> Callable[[PurePosixPath], bool] | None: - """Convert an ignore spec to a single callable filter. - - Args: - ignore: ``None`` (no filtering), a list of glob patterns matched against - the POSIX relative path from the root via ``pathlib.PurePosixPath.match()`` - (right-anchored: ``"*.pyc"`` matches at any depth; ``"sub/*.pyc"`` matches - only inside ``sub/``), or a callable ``(PurePosixPath) -> bool``. - - Returns: - A callable ``(PurePosixPath) -> bool`` returning ``True`` to exclude an entry, - or ``None`` if no filtering is needed. - """ - if ignore is None: - return None - if callable(ignore): - return ignore - patterns = list(ignore) - - def _glob_filter(relative: PurePosixPath) -> bool: - return any(relative.match(pat) for pat in patterns) - - return _glob_filter -``` - -- [ ] **Step 3: Update `_hash_dir` signature and body** - -Replace the entire `_hash_dir` function with: - -```python -def _hash_dir( - path: UPath, - root: UPath, - filter_fn: Callable[[PurePosixPath], bool] | None, - algorithm: str, - file_hasher: FileContentHasherProtocol, -) -> bytes: - """Recursively compute the Merkle hash of a directory. - - Args: - path: The directory to hash (current recursion node). - root: The top-level directory passed to ``BasicDirectoryHasher.hash_directory``. - Used to compute POSIX relative paths for filter evaluation. - filter_fn: Optional filter callable; receives the POSIX relative path from - ``root`` and returns ``True`` to exclude the entry. - algorithm: Hash algorithm name used for structural (entry and node) hashing. - file_hasher: Hasher used to compute ``ContentHash`` for each file leaf. - - Returns: - The raw digest bytes for this directory node. - """ - entries: list[tuple[bytes, bytes]] = [] - - for child in path.iterdir(): - relative = PurePosixPath(child.relative_to(root)) - if filter_fn is not None and filter_fn(relative): - continue - - name_bytes = child.name.encode("utf-8") - - if child.is_symlink(): - target = os.readlink(child) - entry_bytes = b"symlink\x00" + name_bytes + b"\x00" + target.encode("utf-8") - elif child.is_file(): - file_hash = file_hasher.hash_file(child) - entry_bytes = b"file\x00" + name_bytes + b"\x00" + file_hash.digest - elif child.is_dir(): - subdir_digest = _hash_dir(child, root, filter_fn, algorithm, file_hasher) - entry_bytes = b"dir\x00" + name_bytes + b"\x00" + subdir_digest - else: - # Special file (socket, device node, named pipe) — skip silently. - logger.debug("BasicDirectoryHasher: skipping special file %s", child) - continue - - entry_digest = hashlib.new(algorithm, entry_bytes).digest() - entries.append((name_bytes, entry_digest)) - - # Sort byte-wise by name — locale-independent, deterministic. - entries.sort(key=lambda x: x[0]) - - h = hashlib.new(algorithm) - h.update(b"directory\x00") - for _, entry_digest in entries: - h.update(entry_digest) - return h.digest() -``` - -- [ ] **Step 4: Update `BasicDirectoryHasher.hash_directory`** - -In `class BasicDirectoryHasher`, update the `hash_directory` method signature and body. -Replace the method with: - -```python -def hash_directory( - self, - directory_path: PathLike, - ignore: Callable[[PurePosixPath], bool] | list[str] | None = None, -) -> ContentHash: - """Compute the recursive Merkle hash of a directory tree. - - Args: - directory_path: Path to the directory to hash. - ignore: Optional filter. A list of glob patterns matched against the - **POSIX relative path from the root** via ``pathlib.PurePosixPath.match()`` - (right-anchored: ``"*.pyc"`` matches any ``.pyc`` at any depth; - ``"sub/*.pyc"`` matches only ``.pyc`` files directly inside ``sub/``), - or a callable ``(pathlib.PurePosixPath) -> bool`` returning ``True`` - to exclude an entry. Applied at every level of recursion. - - Hash invariant: excluded entries are invisible to the hash. The result - is identical to those entries never existing. The pattern string itself - is not input to the hash. - - Returns: - A ``ContentHash`` with ``method="merkle_{algorithm}"``. - - Raises: - FileNotFoundError: If ``directory_path`` does not exist. - NotADirectoryError: If ``directory_path`` is not a directory. - PermissionError: If the directory is not traversable. - - Example: - >>> from orcapod.hashing.file_hashers import FileHasher - >>> hasher = BasicDirectoryHasher(file_hasher=FileHasher()) - >>> result = hasher.hash_directory("/tmp/mydir") - >>> result.method - 'merkle_sha256' - >>> # Exclude all .pyc files at any depth - >>> result = hasher.hash_directory("/tmp/mydir", ignore=["*.pyc"]) - >>> # Exclude .pyc only inside build/ - >>> result = hasher.hash_directory("/tmp/mydir", ignore=["build/*.pyc"]) - >>> # Custom callable filter (receives PurePosixPath relative to root) - >>> result = hasher.hash_directory("/tmp/mydir", ignore=lambda p: p.name.startswith(".")) - """ - path = UPath(directory_path) - filter_fn = _compile_ignore(ignore) - digest = _hash_dir(path, path, filter_fn, self.algorithm, self.file_hasher) - return ContentHash(method=f"merkle_{self.algorithm}", digest=digest) -``` - -- [ ] **Step 5: Run the full test suite to check** - -```bash -uv run pytest tests/test_hashing/ -v -``` - -Expected: the new test `test_relative_path_pattern_scopes_to_subdirectory` now **PASSES**. -All other tests also pass. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/directory_hashers.py -git commit -m "feat(directory): match ignore patterns against relative POSIX paths" -``` - ---- - -## Task 3: Add hash invariant tests - -**Files:** -- Modify: `tests/test_hashing/test_directory_handler.py` - -- [ ] **Step 1: Add three invariant tests inside `TestBasicDirectoryHasher`** - -After the `test_relative_path_pattern_scopes_to_subdirectory` test added in Task 1, add: - -```python -def test_ignore_none_is_equivalent_to_no_ignore(self, tmp_path): - """hash(dir) == hash(dir, ignore=None) — the new arg in absent form must not shift the hash.""" - d = tmp_path / "d" - d.mkdir() - (d / "file.txt").write_bytes(b"content") - (d / "sub").mkdir() - (d / "sub" / "nested.txt").write_bytes(b"nested") - hasher = BasicDirectoryHasher(file_hasher=FileHasher()) - assert hasher.hash_directory(d) == hasher.hash_directory(d, ignore=None) - -def test_non_matching_filter_is_equivalent_to_no_filter(self, tmp_path): - """A filter that matches nothing must produce the same hash as no filter.""" - d = tmp_path / "d" - d.mkdir() - (d / "file.txt").write_bytes(b"content") - (d / "sub").mkdir() - (d / "sub" / "nested.txt").write_bytes(b"nested") - hasher = BasicDirectoryHasher(file_hasher=FileHasher()) - assert hasher.hash_directory(d) == hasher.hash_directory(d, ignore=["*.nonexistent_xyz"]) - -def test_filter_identity_irrelevance(self, tmp_path): - """Two patterns selecting the same effective file set must produce the same hash.""" - d = tmp_path / "d" - d.mkdir() - (d / "app.py").write_bytes(b"code") - (d / "app.pyc").write_bytes(b"compiled") - hasher = BasicDirectoryHasher(file_hasher=FileHasher()) - # Both patterns exclude only app.pyc — results must be identical - h1 = hasher.hash_directory(d, ignore=["*.pyc"]) - h2 = hasher.hash_directory(d, ignore=["app.pyc"]) - assert h1 == h2 -``` - -- [ ] **Step 2: Run the new tests** - -```bash -uv run pytest tests/test_hashing/test_directory_handler.py::TestBasicDirectoryHasher::test_ignore_none_is_equivalent_to_no_ignore tests/test_hashing/test_directory_handler.py::TestBasicDirectoryHasher::test_non_matching_filter_is_equivalent_to_no_filter tests/test_hashing/test_directory_handler.py::TestBasicDirectoryHasher::test_filter_identity_irrelevance -v -``` - -Expected: all 3 **PASS**. - -- [ ] **Step 3: Run the full test suite** - -```bash -uv run pytest tests/test_hashing/ -v -``` - -Expected: all tests PASS. - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_hashing/test_directory_handler.py -git commit -m "test(directory): add hash invariant tests for ignore= filter (ITL-603)" -``` - ---- - -## Task 4: Update type hint and docstring in `directory_type.py` - -**Files:** -- Modify: `src/orcapod/extension_types/directory_type.py` - -- [ ] **Step 1: Add `PurePosixPath` import** - -At the top of `src/orcapod/extension_types/directory_type.py`, the existing imports include: - -```python -from __future__ import annotations - -import importlib -import json -import os -import warnings -from collections.abc import Callable, Iterable -from typing import TYPE_CHECKING, Any, Self -``` - -Add `from pathlib import PurePosixPath` after the stdlib imports block: - -```python -from __future__ import annotations - -import importlib -import json -import os -import warnings -from collections.abc import Callable, Iterable -from pathlib import PurePosixPath -from typing import TYPE_CHECKING, Any, Self -``` - -- [ ] **Step 2: Update `Directory.__init__` signature and docstring** - -Replace the `__init__` method signature and its docstring. Find: - -```python - def __init__( - self, - *args: Any, - ignore: Callable[[UPath], bool] | Iterable[str] | None = None, - **kwargs: Any, - ) -> None: - super().__init__(*args, **kwargs) - if not self.__wrapped__.exists(): - raise FileNotFoundError( - f"Directory: path does not exist: {self.__wrapped__!r}" - ) - if not self.__wrapped__.is_dir(): - raise NotADirectoryError( - f"Directory: path is not a directory: {self.__wrapped__!r}" - ) - try: - next(iter(self.__wrapped__.iterdir()), None) - except PermissionError as exc: - raise PermissionError( - f"Directory: path is not traversable: {self.__wrapped__!r}" - ) from exc - self._ignore = ignore -``` - -Replace only the signature (first line) with: - -```python - def __init__( - self, - *args: Any, - ignore: Callable[[PurePosixPath], bool] | Iterable[str] | None = None, - **kwargs: Any, - ) -> None: -``` - -(Leave the body unchanged — only the type hint on `ignore` changes.) - -- [ ] **Step 3: Update the `Directory` class docstring's `ignore=` section** - -In the class docstring, find the `Args:` section's `ignore` entry: - -``` - ignore: Optional filter for excluding entries from the content hash. - Accepts an iterable of glob patterns matched against entry names (via - ``fnmatch``), or a callable ``(UPath) -> bool`` returning ``True`` to - exclude an entry. Applied at every level of recursion during hashing. - Defaults to ``None`` (all entries included). -``` - -Replace it with: - -``` - ignore: Optional filter for excluding entries from the content hash. - Accepts an iterable of glob patterns matched against the **POSIX - relative path from the directory root** via - ``pathlib.PurePath.match()`` (right-anchored: ``"*.pyc"`` matches - any ``.pyc`` at any depth; ``"sub/*.pyc"`` matches only ``.pyc`` - files directly inside ``sub/``), or a callable - ``(pathlib.PurePosixPath) -> bool`` receiving the relative path - and returning ``True`` to exclude an entry. Applied at every level - of recursion during hashing. - - Hash invariant: excluded entries are invisible to the hash — the - result is identical to those entries never existing. The pattern - itself is not input to the hash. - - Defaults to ``None`` (all entries included). -``` - -- [ ] **Step 4: Update the `Example:` block in the class docstring** - -Find the existing example block: - -``` - Example: - >>> d = Directory("/tmp/mydata") - >>> str(d) - '/tmp/mydata' - >>> Directory("/tmp/nonexistent") - FileNotFoundError: ... -``` - -Replace with: - -``` - Example: - >>> d = Directory("/tmp/mydata") - >>> str(d) - '/tmp/mydata' - >>> # Exclude compiled Python files at any depth - >>> d = Directory("/tmp/mydata", ignore=["*.pyc", "__pycache__"]) - >>> # Exclude files only inside a specific subdirectory - >>> d = Directory("/tmp/mydata", ignore=["build/*.o"]) - >>> # Custom callable filter: exclude hidden files - >>> d = Directory("/tmp/mydata", ignore=lambda p: p.name.startswith(".")) - >>> Directory("/tmp/nonexistent") - FileNotFoundError: ... -``` - -- [ ] **Step 5: Run the full test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: all tests PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/directory_type.py -git commit -m "docs(directory): update ignore= type hint and docstring for relative-path semantics (ITL-603)" -``` - ---- - -## Task 5: Final verification - -- [ ] **Step 1: Run the complete test suite** - -```bash -uv run pytest tests/ -v -``` - -Expected: all tests PASS. No warnings about unexpected failures. - -- [ ] **Step 2: Verify the test count increased by 4** - -```bash -uv run pytest tests/test_hashing/test_directory_handler.py -v --co -q -``` - -Confirm the 4 new tests appear: -- `TestBasicDirectoryHasher::test_relative_path_pattern_scopes_to_subdirectory` -- `TestBasicDirectoryHasher::test_ignore_none_is_equivalent_to_no_ignore` -- `TestBasicDirectoryHasher::test_non_matching_filter_is_equivalent_to_no_filter` -- `TestBasicDirectoryHasher::test_filter_identity_irrelevance` diff --git a/superpowers/plans/2026-08-07-npipe-204-groupby-operator.md b/superpowers/plans/2026-08-07-npipe-204-groupby-operator.md deleted file mode 100644 index 8a9eecedc..000000000 --- a/superpowers/plans/2026-08-07-npipe-204-groupby-operator.md +++ /dev/null @@ -1,1822 +0,0 @@ -# NPIPE-204 — `GroupBy` Operator + Type-Aware Source Info Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a `GroupBy` operator that reduces N rows sharing a tag tuple into one packet with list-valued members, and fix the `Data` datagram so list-valued provenance no longer crashes `job.run()`. - -**Architecture:** Three independent layers. (1) `Data` derives `_source_*` Arrow/Python types from the stored value instead of hard-coding `large_string`/`str` — this alone repairs the already-shipped `MergeJoin`. (2) A shared `arrow_utils` helper folds N members' system-tag values into one scalar using SHA-based primitives already in the codebase. (3) `GroupBy` partitions by tag values, sorts members deterministically, list-wraps everything except the group keys, and folds the system tags. `Batch` is semantically untouched — it only picks up the provenance fix. - -**Tech Stack:** Python 3.11+, PyArrow, Polars, pytest, `uv run` for all commands, Delta Lake (`DeltaTableDatabase`) for job-level tests. - -**Spec:** `superpowers/specs/2026-08-07-npipe-204-batch-group-by-design.md` - -**Linear:** NPIPE-204 (status already *In Progress*). Branch `arnoldb/npipe-204-orcapod-add-tag-based-grouping-to-batch-manyone-reduction` is already checked out. - ---- - -## Background an implementer needs - -**The bug, minimally.** A `Data` datagram whose source info holds a list crashes on `as_table`: - -```python -data.source_info() # {'probe': None, 'path': ['s0', 's1']} -data.schema(columns={"source": True}) # '_source_path': str <- WRONG -data.as_table(columns={"source": True}) # ArrowTypeError: Expected bytes, got a 'list' object -``` - -Two operators already produce this state: `MergeJoin` (carries source columns as parallel lists when merging colliding data columns, `merge_join.py:262`) and `Batch` (list-wraps every column). Neither is caught by existing tests because all operator tests stop at `op.process(stream)` + `as_table()`; the crash happens in `StaticOutputOperatorPod._materialize_to_stream`, only reached via `job.run()`. - -**Column vocabulary.** For a stream with tag `subject` and data `path`, `stream.as_table(columns={"source": True, "system_tags": True})` yields exactly: - -``` -subject # user tag -path # data -_tag_source_id:: # system tag, large_string -_tag_record_id:: # system tag, binary(16) -_source_path # provenance, large_string -``` - -Note `_context_key` is **absent** — no operator sees it. Do not add it. - -**`ArrowTableStream` fills in missing source columns.** If the output table has a data column with no matching `_source_`, the stream creates one with value `None` and type `large_string`. Verified. So `GroupBy` does not need to synthesize source columns for the tag columns it promotes to data — but `unary_output_schema` must still *predict* them. - ---- - -## File Structure - -| File | Responsibility | Action | -|---|---|---| -| `src/orcapod/core/datagrams/tag_data.py` | `Data` source-info type derivation | Modify | -| `src/orcapod/utils/polars_data_utils.py` | Delete dead `add_source_info` | Modify | -| `src/orcapod/utils/arrow_utils.py` | Shared system-tag fold helper | Modify | -| `src/orcapod/core/operators/batch.py` | Provenance fix only | Modify | -| `src/orcapod/core/operators/group_by.py` | The new operator | **Create** | -| `src/orcapod/core/operators/__init__.py` | Export | Modify | -| `src/orcapod/pipeline/serialization.py` | Operator registry | Modify | -| `src/orcapod/core/streams/base.py` | `.group_by()` fluent method | Modify | -| `tests/test_core/datagrams/test_data_source_info_types.py` | Part 1 unit tests | **Create** | -| `tests/test_utils/test_arrow_utils.py` | Fold helper + cross-process stability | Modify | -| `tests/test_core/operators/test_group_by.py` | Operator-level tests | **Create** | -| `tests/test_pipeline/test_aggregation_job.py` | Job-level tests | **Create** | -| `CLAUDE.md` + `.zed/rules` | Doc updates | Modify | -| `DESIGN_ISSUES.md` | U1 resolution note | Modify | - ---- - -## Task 1: Type-aware source info in `Data` - -**Files:** -- Modify: `src/orcapod/core/datagrams/tag_data.py` -- Test: `tests/test_core/datagrams/test_data_source_info_types.py` (create) - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_core/datagrams/test_data_source_info_types.py`: - -```python -"""Source-info values may be lists, not just scalar strings. - -Many->one operators (GroupBy, MergeJoin) produce one provenance token per -member. `Data` must represent those without collapsing or crashing. -""" - -from __future__ import annotations - -import pyarrow as pa - -from orcapod.core.datagrams import Data - - -def _data_with_mixed_source_info() -> Data: - """Data with one list-valued and one scalar-null source token.""" - return Data( - {"probe": [0, 1], "path": ["a", "b"]}, - source_info={"probe": None, "path": ["s0", "s1"]}, - ) - - -class TestListValuedSourceInfo: - def test_schema_reports_list_type_for_list_valued_token(self): - data = _data_with_mixed_source_info() - schema = data.schema(columns={"source": True}) - assert schema["_source_path"] == list[str] - - def test_schema_reports_str_for_none_token(self): - data = _data_with_mixed_source_info() - schema = data.schema(columns={"source": True}) - assert schema["_source_probe"] is str - - def test_as_table_round_trips_list_valued_token(self): - data = _data_with_mixed_source_info() - table = data.as_table(columns={"source": True}) - assert table.schema.field("_source_path").type == pa.large_list( - pa.large_string() - ) - assert table.column("_source_path").to_pylist() == [["s0", "s1"]] - - def test_as_table_keeps_none_token_as_large_string(self): - data = _data_with_mixed_source_info() - table = data.as_table(columns={"source": True}) - assert table.schema.field("_source_probe").type == pa.large_string() - assert table.column("_source_probe").to_pylist() == [None] - - def test_empty_list_token_defaults_to_list_of_string(self): - data = Data({"path": ["a"]}, source_info={"path": []}) - table = data.as_table(columns={"source": True}) - assert table.schema.field("_source_path").type == pa.large_list( - pa.large_string() - ) - - def test_scalar_token_unchanged(self): - """Existing scalar behavior must not regress.""" - data = Data({"path": "a"}, source_info={"path": "src::row_0::path"}) - table = data.as_table(columns={"source": True}) - assert table.schema.field("_source_path").type == pa.large_string() - assert data.schema(columns={"source": True})["_source_path"] is str - - def test_arrow_table_construction_recovers_list_token(self): - """Data built from an Arrow table keeps list-valued source info.""" - table = pa.table({ - "path": pa.array([["a", "b"]], pa.list_(pa.large_string())), - "_source_path": pa.array([["s0", "s1"]], pa.list_(pa.large_string())), - }) - data = Data(table) - assert data.source_info()["path"] == ["s0", "s1"] - assert data.as_table(columns={"source": True}).column( - "_source_path" - ).to_pylist() == [["s0", "s1"]] -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_core/datagrams/test_data_source_info_types.py -v -``` - -Expected: `test_schema_reports_list_type_for_list_valued_token` FAILS (`assert str == list[str]`), and the four `as_table` tests FAIL with `ArrowTypeError: Expected bytes, got a 'list' object`. `test_schema_reports_str_for_none_token` and `test_scalar_token_unchanged` should already PASS. - -- [ ] **Step 3: Add the type-derivation helpers** - -In `src/orcapod/core/datagrams/tag_data.py`, immediately above `class Data(Datagram):` (currently line 240), add: - -```python -# A provenance token is a string, an unknown (None), or — for many->one -# operators such as GroupBy and MergeJoin — a list of tokens, one per member. -SourceInfoValue = str | None | list["SourceInfoValue"] - - -def _source_info_arrow_type(value: "SourceInfoValue") -> "pa.DataType": - """Derive the Arrow type for a single source-info value. - - Scalars and unknowns map to ``large_string``; lists map to ``large_list`` - of their element type, recursively. An empty list defaults to - ``large_list(large_string)``. - - Args: - value: The stored provenance token. - - Returns: - The Arrow type to declare for this value. - """ - import pyarrow as _pa - - if isinstance(value, (list, tuple)): - if not value: - return _pa.large_list(_pa.large_string()) - return _pa.large_list(_source_info_arrow_type(value[0])) - return _pa.large_string() - - -def _source_info_python_type(value: "SourceInfoValue") -> type: - """Derive the Python type for a single source-info value. - - Mirrors ``_source_info_arrow_type`` for the ``Schema`` representation. - - Args: - value: The stored provenance token. - - Returns: - ``str`` for scalars and unknowns, ``list[...]`` for lists. - """ - if isinstance(value, (list, tuple)): - if not value: - return list[str] - return list[_source_info_python_type(value[0])] # type: ignore[misc] - return str -``` - -- [ ] **Step 4: Use the helpers in `_ensure_source_info_table`** - -Replace the body of `_ensure_source_info_table` (currently `tag_data.py:330-347`): - -```python - def _ensure_source_info_table(self) -> "pa.Table": - if self._source_info_table is None: - import pyarrow as _pa - - if self._source_info: - prefixed = { - f"{constants.SOURCE_PREFIX}{k}": v - for k, v in self._source_info.items() - } - schema = _pa.schema( - [ - _pa.field(k, _source_info_arrow_type(v)) - for k, v in prefixed.items() - ] - ) - self._source_info_table = _pa.Table.from_pylist( - [prefixed], schema=schema - ) - else: - self._source_info_table = _pa.table({}) - return self._source_info_table -``` - -- [ ] **Step 5: Use the helper in `Data.schema`** - -In `Data.schema` (currently `tag_data.py:384-395`), replace the `if column_config.source:` block: - -```python - if column_config.source: - for key in super().keys(): - schema[f"{constants.SOURCE_PREFIX}{key}"] = _source_info_python_type( - self._source_info.get(key) - ) -``` - -- [ ] **Step 6: Widen the type annotations** - -Four annotation sites in `tag_data.py`, all mechanical — no logic change: - -1. `Data.__init__` signature (line ~257): - `source_info: "Mapping[str, str | None] | None" = None` - → `source_info: "Mapping[str, SourceInfoValue] | None" = None` -2. Arrow-path assignment (line ~296): - `self._source_info: dict[str, str | None] = {` - → `self._source_info: dict[str, SourceInfoValue] = {` -3. Dict-path local (line ~309): - `contained_source_info: dict[str, str | None] = {` - → `contained_source_info: dict[str, SourceInfoValue] = {` -4. `source_info()` return (line ~353) and `with_source_info()` kwargs (line ~357): - `def source_info(self) -> "dict[str, str | None]":` - → `def source_info(self) -> "dict[str, SourceInfoValue]":` - `def with_source_info(self, **source_info: "str | None") -> Self:` - → `def with_source_info(self, **source_info: "SourceInfoValue") -> Self:` - -- [ ] **Step 7: Run the test to verify it passes** - -```bash -uv run pytest tests/test_core/datagrams/test_data_source_info_types.py -v -``` - -Expected: all 7 PASS. - -- [ ] **Step 8: Verify the MergeJoin crash is gone** - -This is the payoff — an already-shipped operator that was broken. - -```bash -uv run pytest tests/test_core/ tests/test_utils/ -x -q -``` - -Expected: no failures, no new errors. - -- [ ] **Step 9: Commit** - -```bash -git add src/orcapod/core/datagrams/tag_data.py tests/test_core/datagrams/test_data_source_info_types.py -git commit -m "fix(datagrams): derive source-info column types from value (NPIPE-204) - -_ensure_source_info_table and Data.schema hard-coded large_string / str for -every _source_* field, so any operator producing a list-valued provenance -token crashed with ArrowTypeError inside job.run(). MergeJoin already does -this (merge_join.py:262) and was silently broken. - -Types are now derived from the stored value. None still maps to -large_string, so unknown-provenance columns are unchanged. No pipeline-DB -schema bump: a node's source-column type is fixed by its own output schema. - -Refs DESIGN_ISSUES U1. -NPIPE-204" -``` - ---- - -## Task 2: Delete the dead `polars_data_utils.add_source_info` - -**Files:** -- Modify: `src/orcapod/utils/polars_data_utils.py` - -Third site with the same hard-coded assumption (`dtype=pl.String()`, line 119). It has **zero callers** in `src/` and no test coverage — the tests that import `add_source_info` import it from `arrow_utils`. It also has a latent shadowing bug: `source_column` is rebound to a `pl.Series` inside the per-column loop, so from the second column onward it formats `f"{}::{col}"`. - -- [ ] **Step 1: Confirm it is unreferenced** - -```bash -grep -rn "polars_data_utils" --include=*.py src/ tests/ -grep -rn "add_source_info" --include=*.py src/ tests/ -``` - -Expected: the only `polars_data_utils` references are `drop_system_columns` (from `data_frame_source.py:57` and `polling_source.py:454`). Every `add_source_info` reference resolves to `arrow_utils`. If either expectation fails, **stop** and report — do not delete. - -- [ ] **Step 2: Delete the function** - -Remove the entire `def add_source_info(...)` definition from `src/orcapod/utils/polars_data_utils.py` (starts line 93, ends at the `return df` around line 125). Leave every other function in the module untouched. - -- [ ] **Step 3: Verify nothing broke** - -```bash -uv run pytest tests/ -q -x -``` - -Expected: full suite passes. - -- [ ] **Step 4: Commit** - -```bash -git add src/orcapod/utils/polars_data_utils.py -git commit -m "chore(utils): delete dead polars_data_utils.add_source_info (NPIPE-204) - -Zero callers in src/, no test coverage, and a latent shadowing bug where -source_column is rebound to a pl.Series inside the per-column loop. It was -also a third site hard-coding a scalar string type for source info. - -Refs DESIGN_ISSUES U1. -NPIPE-204" -``` - ---- - -## Task 3: Shared system-tag fold helper - -**Files:** -- Modify: `src/orcapod/utils/arrow_utils.py` -- Test: `tests/test_utils/test_arrow_utils.py` - -`_build_record_id_preimage` (`core/nodes/function_node.py:82`) hashes system-tag columns directly, so they must be scalar. A many→one operator must fold N members' values into one, preserving each column's Arrow type. - -**Both primitives are reused from the codebase and are SHA-based, therefore stable across processes.** Never use `hash()` or set iteration order here — a fold using either would pass every same-process test and then miss the cache on every new driver run. - -- [ ] **Step 1: Write the failing test** - -Append to `tests/test_utils/test_arrow_utils.py`: - -```python -# --------------------------------------------------------------------------- -# fold_system_tag_values -# --------------------------------------------------------------------------- - - -class TestFoldSystemTagValues: - """Folding N members' system-tag values into one scalar (NPIPE-204). - - The expected digests below are hard-coded on purpose. A fold that used - hash() or set-iteration order would still be self-consistent within one - process; pinning the values is what catches it. - """ - - SOURCE_COL = "_tag_source_id::abc123" - RECORD_COL = "_tag_record_id::abc123" - - RIDS = [ - bytes.fromhex("0102030405060708090a0b0c0d0e0f10"), - bytes.fromhex("1112131415161718191a1b1c1d1e1f20"), - ] - EXPECTED_RID = bytes.fromhex("853be16a3f38565f8ced039f84fdbea6") - EXPECTED_SID = ( - "7916442d59841140bedf6c1f5dcc1304ae9fce0ba885765c06e511086b85da2e" - ) - - def test_record_id_folds_to_16_bytes(self): - from orcapod.utils.arrow_utils import fold_system_tag_values - - result = fold_system_tag_values(self.RECORD_COL, self.RIDS) - assert isinstance(result, bytes) - assert len(result) == 16 - - def test_record_id_digest_is_pinned(self): - from orcapod.utils.arrow_utils import fold_system_tag_values - - assert fold_system_tag_values(self.RECORD_COL, self.RIDS) == self.EXPECTED_RID - - def test_source_id_digest_is_pinned(self): - from orcapod.utils.arrow_utils import fold_system_tag_values - - result = fold_system_tag_values(self.SOURCE_COL, ["src_a", "src_b"]) - assert result == self.EXPECTED_SID - - def test_order_matters(self): - """Member order is part of the identity, matching the data lists.""" - from orcapod.utils.arrow_utils import fold_system_tag_values - - forward = fold_system_tag_values(self.RECORD_COL, self.RIDS) - reverse = fold_system_tag_values(self.RECORD_COL, list(reversed(self.RIDS))) - assert forward != reverse - - def test_single_member_is_still_folded(self): - """A one-member group folds rather than passing the value through.""" - from orcapod.utils.arrow_utils import fold_system_tag_values - - result = fold_system_tag_values(self.RECORD_COL, self.RIDS[:1]) - assert isinstance(result, bytes) and len(result) == 16 - assert result != self.RIDS[0] - - def test_none_members_are_tolerated(self): - from orcapod.utils.arrow_utils import fold_system_tag_values - - assert isinstance( - fold_system_tag_values(self.RECORD_COL, [None, self.RIDS[0]]), bytes - ) - assert isinstance( - fold_system_tag_values(self.SOURCE_COL, [None, "src_a"]), str - ) - - def test_digest_is_stable_across_processes(self): - """Fresh interpreter, therefore fresh PYTHONHASHSEED. - - This is the test that catches a fold built on hash() or set order: - such a fold is self-consistent within one process and only diverges - on a new driver run. - """ - import subprocess - import sys - - script = ( - "from orcapod.utils.arrow_utils import fold_system_tag_values\n" - "rids = [bytes.fromhex('0102030405060708090a0b0c0d0e0f10'),\n" - " bytes.fromhex('1112131415161718191a1b1c1d1e1f20')]\n" - "print(fold_system_tag_values('_tag_record_id::abc123', rids).hex())\n" - "print(fold_system_tag_values('_tag_source_id::abc123', ['src_a','src_b']))\n" - ) - out = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - text=True, - check=True, - ).stdout.split() - assert out[0] == self.EXPECTED_RID.hex() - assert out[1] == self.EXPECTED_SID -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_utils/test_arrow_utils.py::TestFoldSystemTagValues -v -``` - -Expected: all FAIL with `ImportError: cannot import name 'fold_system_tag_values'`. - -- [ ] **Step 3: Implement the helper** - -In `src/orcapod/utils/arrow_utils.py`, add immediately after `append_to_system_tags` (currently ends line 1157): - -```python -# Fixed namespace for aggregated record IDs produced by many->one operators. -# Mirrors _SOURCE_RECORD_ID_NAMESPACE in core/sources/stream_builder.py. -# Computed value: uuid.UUID('96411bfc-d3ba-5395-ba6f-5bb5726f18ad') -_AGGREGATED_RECORD_ID_NAMESPACE = uuid.uuid5( - uuid.NAMESPACE_URL, - "https://orcapod.org/namespaces/aggregated-record-id", -) - - -def fold_system_tag_values( - column_name: str, values: "Sequence[Any]" -) -> "str | bytes": - """Fold a group's system-tag values into one scalar of the same type. - - Many->one operators must emit scalar system tags, because - ``_build_record_id_preimage`` (``core/nodes/function_node.py``) hashes - those columns directly to derive a record's identity. Each column folds - independently over its own ordered member values. - - Both digests are SHA-based and therefore stable across processes. Never - substitute ``hash()`` or a set-based construction: orcapod uses the result - as a cache key, so a per-process digest would miss the cache on every new - driver run while looking correct in a single-process test. - - Member order is significant — it matches the order of the list-valued data - columns the folded tag accompanies. - - Args: - column_name: The system-tag column name, used to select the fold. Names - starting with ``constants.SYSTEM_TAG_RECORD_ID_PREFIX`` fold to - ``binary(16)``; everything else folds to a hex string. - values: The group's member values, in emission order. - - Returns: - 16 raw bytes for a record_id column, a 64-character hex string - otherwise. - """ - if column_name.startswith(constants.SYSTEM_TAG_RECORD_ID_PREFIX): - name = constants.BLOCK_SEPARATOR.join( - "" if v is None else v.hex() for v in values - ) - return uuid.uuid5(_AGGREGATED_RECORD_ID_NAMESPACE, name).bytes - return combine_hashes( - *["" if v is None else str(v) for v in values], order=False - ) -``` - -Add the imports at the top of `arrow_utils.py` if not already present: - -```python -import uuid - -from orcapod.hashing.hash_utils import combine_hashes -``` - -If importing `combine_hashes` at module scope creates a circular import, move it inside the function body instead and note why. - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/test_utils/test_arrow_utils.py::TestFoldSystemTagValues -v -``` - -Expected: all 7 PASS, including `test_digest_is_stable_across_processes`. - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/utils/arrow_utils.py tests/test_utils/test_arrow_utils.py -git commit -m "feat(arrow_utils): add fold_system_tag_values for many-to-one operators (NPIPE-204) - -System tags must stay scalar because _build_record_id_preimage hashes them -directly. Each column folds independently over its ordered member values: -record_id via uuid5 (matching stream_builder._make_record_id), source_id via -combine_hashes. Both SHA-based, so the digest is stable across processes -- -pinned by a subprocess test, since a hash()-based fold would look correct -within one process and miss the cache on every new driver run. - -NPIPE-204" -``` - ---- - -## Task 4: `Batch` provenance fix - -**Files:** -- Modify: `src/orcapod/core/operators/batch.py:43-102` -- Test: `tests/test_core/operators/test_operators.py` - -`Batch`'s partitioning, its list-valued tag columns, and its streaming `async_execute` all stay exactly as they are. The only change: system-tag columns fold to scalars instead of being list-wrapped, and `_source_*` columns keep their list form (now representable thanks to Task 1). - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_core/operators/test_operators.py`, inside `class TestBatchBehavior`: - -```python - def test_batch_system_tags_are_scalar(self): - """System tags must stay scalar -- record identity hashes them directly.""" - from orcapod.core.sources import ArrowTableSource - from orcapod.system_constants import constants - - table = pa.table({ - "animal": ["cat", "dog"], - "weight": [4.0, 12.0], - }) - source = ArrowTableSource(table, tag_columns=["animal"], infer_nullable=True) - out = Batch(batch_size=0).process(source) - result = out.as_table(columns={"source": True, "system_tags": True}) - - sys_cols = [ - c for c in result.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ] - assert sys_cols, "expected system tag columns on the batched output" - for col in sys_cols: - assert not pa.types.is_list(result.schema.field(col).type) - assert not pa.types.is_large_list(result.schema.field(col).type) - - def test_batch_source_columns_are_lists(self): - """Provenance stays per-member rather than collapsing.""" - from orcapod.core.sources import ArrowTableSource - from orcapod.system_constants import constants - - table = pa.table({ - "animal": ["cat", "dog"], - "weight": [4.0, 12.0], - }) - source = ArrowTableSource(table, tag_columns=["animal"], infer_nullable=True) - out = Batch(batch_size=0).process(source) - result = out.as_table(columns={"source": True, "system_tags": True}) - - src_col = f"{constants.SOURCE_PREFIX}weight" - assert src_col in result.column_names - assert len(result.column(src_col).to_pylist()[0]) == 2 -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_core/operators/test_operators.py::TestBatchBehavior -v -``` - -Expected: `test_batch_system_tags_are_scalar` FAILS (system tag columns are `list<...>`). `test_batch_source_columns_are_lists` should already PASS. - -- [ ] **Step 3: Rewrite `unary_static_process`** - -Replace `Batch.unary_static_process` (`batch.py:43-82`) with: - -```python - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: - """Group rows into fixed-size batches, list-wrapping their values. - - Tag and data columns become list-valued. Source-info columns become - list-valued too, one element per batch member. System-tag columns are - folded to a scalar instead — record identity hashes them directly, so - they must not become lists. - - Args: - stream: The upstream stream. - - Returns: - A stream with one row per batch. - """ - table = stream.as_table(columns={"source": True, "system_tags": True}) - - tag_columns, _ = stream.keys() - - system_tag_columns = tuple( - c - for c in table.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ) - member_columns = tuple( - c for c in table.column_names if c not in system_tag_columns - ) - - data_list = table.to_pylist() - - batches: list[list[dict[str, Any]]] = [] - next_batch: list[dict[str, Any]] = [] - - for entry in data_list: - next_batch.append(entry) - if self.batch_size > 0 and len(next_batch) >= self.batch_size: - batches.append(next_batch) - next_batch = [] - - if next_batch and not self.drop_partial_batch: - batches.append(next_batch) - - batched_data = [ - { - **{c: [m[c] for m in members] for c in member_columns}, - **{ - c: arrow_utils.fold_system_tag_values(c, [m[c] for m in members]) - for c in system_tag_columns - }, - } - for members in batches - ] - - input_fields = {f.name: f for f in table.schema} - batched_schema = pa.schema( - [ - pa.field(c, pa.list_(input_fields[c].type), nullable=False) - if c in member_columns - else input_fields[c] - for c in table.column_names - ] - ) - batched_table = pa.Table.from_pylist(batched_data, schema=batched_schema) - - n_char = self.orcapod_config.hashing.system_tag_n_char - batched_table = arrow_utils.append_to_system_tags( - batched_table, stream.pipeline_hash().to_hex(n_char) - ) - - return ArrowTableStream( - batched_table, - tag_columns=tag_columns, - data_context=stream.data_context, - ) -``` - -Add these imports to the top of `batch.py`: - -```python -from orcapod.system_constants import constants -from orcapod.utils import arrow_utils -``` - -- [ ] **Step 4: Update `unary_output_schema` to mirror it** - -Replace `Batch.unary_output_schema` (`batch.py:84-102`): - -```python - def unary_output_schema( - self, - stream: StreamProtocol, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Predict the batched output schemas without batching. - - Every user tag, data, and source column becomes ``list[T]``. System - tag columns keep their scalar type and gain a ``::{pipeline_hash}`` - name suffix. - - Args: - stream: The upstream stream. - columns: Column inclusion config. - all_info: Include all info columns. - - Returns: - A ``(tag_schema, data_schema)`` tuple. - """ - tag_types, data_types = stream.output_schema(columns=columns, all_info=all_info) - n_char = self.orcapod_config.hashing.system_tag_n_char - suffix = stream.pipeline_hash().to_hex(n_char) - - batched_tag_types: dict[str, Any] = {} - for name, col_type in tag_types.items(): - if name.startswith(constants.SYSTEM_TAG_PREFIX): - batched_tag_types[ - f"{name}{constants.BLOCK_SEPARATOR}{suffix}" - ] = col_type - else: - batched_tag_types[name] = list[col_type] - - batched_data_types = {k: list[v] for k, v in data_types.items()} - - return Schema(batched_tag_types), Schema(batched_data_types) -``` - -- [ ] **Step 5: Run the tests to verify they pass** - -```bash -uv run pytest tests/test_core/operators/ -v -``` - -Expected: all PASS, including the pre-existing `TestBatchBehavior` cases (`test_batch_groups_rows`, `test_batch_drop_partial`, `test_batch_output_lineage`, `test_batch_size_zero_returns_single_batch`, `test_negative_batch_size_raises`). - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/operators/batch.py tests/test_core/operators/test_operators.py -git commit -m "fix(batch): fold system tags to scalar instead of list-wrapping (NPIPE-204) - -Batch list-wrapped every column including the system tags, but record -identity hashes those columns directly, so they must stay scalar. They now -fold via fold_system_tag_values and gain a ::{pipeline_hash} name suffix, -mirroring the name-extending rule joins already use. - -Partitioning, list-valued tag columns, and the streaming async_execute path -are unchanged. - -NPIPE-204" -``` - ---- - -## Task 5: The `GroupBy` operator - -**Files:** -- Create: `src/orcapod/core/operators/group_by.py` -- Test: `tests/test_core/operators/test_group_by.py` (create) - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_core/operators/test_group_by.py`: - -```python -"""Tests for the GroupBy operator — many->one reduction keyed on tag values.""" - -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.operators import GroupBy -from orcapod.core.sources import ArrowTableSource -from orcapod.errors import InputValidationError -from orcapod.system_constants import constants - - -@pytest.fixture -def session_table() -> pa.Table: - """Two sessions x two probes: the common-clock shape from NPIPE-204.""" - return pa.table({ - "subject": ["G", "G", "G", "G"], - "date": ["d1", "d1", "d2", "d2"], - "probe": [1, 0, 1, 0], - "path": ["b", "a", "d", "c"], - }) - - -@pytest.fixture -def session_source(session_table) -> ArrowTableSource: - return ArrowTableSource( - session_table, - tag_columns=["subject", "date", "probe"], - infer_nullable=True, - ) - - -class TestGroupByShape: - def test_one_row_per_distinct_key(self, session_source): - out = GroupBy(by=["subject", "date"]).process(session_source) - assert len(out.as_table()) == 2 - - def test_group_keys_are_scalar_tags(self, session_source): - op = GroupBy(by=["subject", "date"]) - out = op.process(session_source) - tag_cols, _ = out.keys() - assert set(tag_cols) == {"subject", "date"} - assert out.as_table().column("subject").to_pylist() == ["G", "G"] - - def test_non_key_tags_promoted_to_list_data(self, session_source): - out = GroupBy(by=["subject", "date"]).process(session_source) - _, data_cols = out.keys() - assert "probe" in data_cols - assert out.as_table().column("probe").to_pylist() == [[0, 1], [0, 1]] - - def test_data_columns_are_lists(self, session_source): - out = GroupBy(by=["subject", "date"]).process(session_source) - assert out.as_table().column("path").to_pylist() == [["a", "b"], ["c", "d"]] - - def test_source_columns_are_lists(self, session_source): - out = GroupBy(by=["subject", "date"]).process(session_source) - table = out.as_table(columns={"source": True}) - assert len(table.column(f"{constants.SOURCE_PREFIX}path").to_pylist()[0]) == 2 - - def test_system_tags_are_scalar_and_renamed(self, session_source): - out = GroupBy(by=["subject", "date"]).process(session_source) - table = out.as_table(columns={"system_tags": True}) - sys_cols = [ - c for c in table.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ] - assert sys_cols - for col in sys_cols: - field_type = table.schema.field(col).type - assert not pa.types.is_list(field_type) - assert not pa.types.is_large_list(field_type) - # name-extended: original "::" plus "::" - assert col.count(constants.BLOCK_SEPARATOR) >= 2 - - -class TestGroupByOrdering: - def test_members_sorted_by_non_key_tags(self, session_source): - """probe=[1,0] on input must emit as [0,1].""" - out = GroupBy(by=["subject", "date"]).process(session_source) - assert out.as_table().column("probe").to_pylist()[0] == [0, 1] - - def test_row_order_does_not_affect_output(self, session_table): - """Same rows, shuffled, must produce a byte-identical table.""" - shuffled = session_table.take([3, 1, 2, 0]) - - def run(tbl): - src = ArrowTableSource( - tbl, tag_columns=["subject", "date", "probe"], infer_nullable=True - ) - return GroupBy(by=["subject", "date"]).process(src).as_table() - - assert run(session_table).equals(run(shuffled)) - - def test_falls_back_to_record_id_when_key_covers_all_tags(self): - """by covering every tag leaves no non-key tag to sort on.""" - table = pa.table({"subject": ["G", "G"], "path": ["b", "a"]}) - src = ArrowTableSource(table, tag_columns=["subject"], infer_nullable=True) - out = GroupBy(by=["subject"]).process(src) - assert len(out.as_table()) == 1 - assert sorted(out.as_table().column("path").to_pylist()[0]) == ["a", "b"] - - -class TestGroupByValidation: - def test_empty_by_raises(self): - with pytest.raises(ValueError, match="at least one"): - GroupBy(by=[]) - - def test_unknown_column_raises(self, session_source): - op = GroupBy(by=["subject", "nonexistent"]) - with pytest.raises(InputValidationError, match="nonexistent"): - op.process(session_source) - - def test_data_column_as_key_raises(self, session_source): - """Grouping on a data column is not allowed -- keys must be tags.""" - op = GroupBy(by=["path"]) - with pytest.raises(InputValidationError, match="path"): - op.process(session_source) - - -class TestGroupByEmptyInput: - def test_empty_input_yields_zero_groups(self): - table = pa.table({ - "subject": pa.array([], pa.large_string()), - "path": pa.array([], pa.large_string()), - }) - from orcapod.core.streams import ArrowTableStream - - stream = ArrowTableStream(table, tag_columns=["subject"]) - out = GroupBy(by=["subject"]).process(stream) - assert len(out.as_table()) == 0 - - -class TestGroupByIdentity: - def test_identity_structure_includes_by(self): - assert GroupBy(by=["a"]).identity_structure() != GroupBy( - by=["b"] - ).identity_structure() - - def test_to_config_round_trip(self): - op = GroupBy(by=["subject", "date"]) - config = op.to_config() - # A list, not a tuple, so the config stays JSON-serializable. - assert config["config"]["by"] == ["subject", "date"] - rebuilt = GroupBy.from_config(config) - assert rebuilt.identity_structure() == op.identity_structure() - - def test_to_config_is_json_serializable(self): - import json - - json.dumps(GroupBy(by=["subject", "date"]).to_config()["config"]) -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_core/operators/test_group_by.py -v -``` - -Expected: all FAIL at collection with `ImportError: cannot import name 'GroupBy'`. - -- [ ] **Step 3: Create the operator** - -**Do not write an `async_execute` override.** `UnaryOperator.async_execute` -(`core/operators/base.py:71`) already collects the full input before calling -`static_process`, which is exactly the barrier `GroupBy` needs — no group can be emitted -before the input channel closes, because any row not yet seen could belong to a group -already started. Task 7 adds a test asserting the override is absent. - -Create `src/orcapod/core/operators/group_by.py`: - -```python -"""GroupBy operator — many->one reduction keyed on tag values.""" - -from __future__ import annotations - -import logging -from collections.abc import Collection -from typing import TYPE_CHECKING, Any - -from orcapod.core.operators.base import UnaryOperator -from orcapod.core.streams import ArrowTableStream -from orcapod.errors import InputValidationError -from orcapod.protocols.core_protocols import StreamProtocol -from orcapod.system_constants import constants -from orcapod.types import ColumnConfig, Schema -from orcapod.utils import arrow_utils -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import pyarrow as pa -else: - pa = LazyModule("pyarrow") - -logger = logging.getLogger(__name__) - - -class GroupBy(UnaryOperator): - """Reduce rows sharing a tag tuple into one packet with list-valued members. - - This is the only many->one operator. Every other operator preserves one - row per tag; ``GroupBy`` collapses N rows into one, which is what lets a - downstream pod receive a whole group at once (for example, all of a - recording session's per-probe result parquets). - - Given tags ``(subject, date, probe)`` and data ``(path)``, grouping by - ``["subject", "date"]`` emits one row per distinct ``(subject, date)``: - - * ``subject`` and ``date`` stay scalar and remain the output's tag columns - * ``probe`` becomes a list-valued **data** column, so a consumer can tell - which member each list element came from - * ``path`` becomes list-valued - * ``_source_*`` columns become list-valued, one element per member - * system-tag columns fold to a scalar digest and gain a - ``::{pipeline_hash}`` name suffix - - Members are sorted by their non-group-key tag values, so the emitted lists - are stable across runs. This matters because orcapod hashes those lists to - build the cache key — an unsorted list would make an identical member set - hash differently and trigger a spurious recompute. - - Contrast with ``Batch``, which partitions by row count for throughput and - keeps its tag columns as list-valued tags. - - Args: - by: Tag column names to group on. Must be non-empty and must all be - tag columns of the input stream. - """ - - def __init__(self, by: Collection[str], **kwargs: Any) -> None: - by_tuple = tuple(by) - if not by_tuple: - raise ValueError("GroupBy requires at least one column in `by`.") - self.by = by_tuple - super().__init__(**kwargs) - - def identity_structure(self) -> Any: - return (self.__class__.__name__, self.by) - - def to_config(self) -> dict[str, Any]: - """Serialize this GroupBy operator to a config dict. - - ``by`` is emitted as a list rather than a tuple so the config stays - JSON-serializable; ``__init__`` normalizes it back to a tuple. - - Returns: - A dict with ``class_name``, ``module_path``, and ``config`` keys, - where ``config`` contains ``by``. - """ - config = super().to_config() - config["config"] = {"by": list(self.by)} - return config - - # ------------------------------------------------------------------ - # Validation - # ------------------------------------------------------------------ - - def validate_unary_input(self, stream: StreamProtocol) -> None: - """Verify every grouping column is a tag column of the input. - - Args: - stream: The upstream stream to validate. - - Raises: - InputValidationError: If any name in ``by`` is not a tag column. - """ - tag_columns, data_columns = stream.keys() - missing = [c for c in self.by if c not in tag_columns] - if missing: - raise InputValidationError( - f"GroupBy: {missing} are not tag columns of the input stream. " - f"Available tag columns: {list(tag_columns)}. " - f"(Data columns cannot be grouping keys: {list(data_columns)})" - ) - - # ------------------------------------------------------------------ - # Processing - # ------------------------------------------------------------------ - - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: - """Partition rows by group key and emit one row per group. - - Args: - stream: The upstream stream. - - Returns: - A stream with one row per distinct group-key tuple. - """ - table = stream.as_table(columns={"source": True, "system_tags": True}) - tag_columns, _ = stream.keys() - - system_tag_columns = tuple( - c - for c in table.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ) - member_columns = tuple( - c - for c in table.column_names - if c not in self.by and c not in system_tag_columns - ) - # Non-key user tags give a total order within a group: tags are unique - # within a stream. When `by` covers every tag, fall back to record_id. - sort_columns = tuple(c for c in tag_columns if c not in self.by) - record_id_column = next( - ( - c - for c in system_tag_columns - if c.startswith(constants.SYSTEM_TAG_RECORD_ID_PREFIX) - ), - None, - ) - - groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {} - for row in table.to_pylist(): - groups.setdefault(tuple(row[c] for c in self.by), []).append(row) - - grouped_rows: list[dict[str, Any]] = [] - for key, members in groups.items(): - if sort_columns: - members.sort( - key=lambda r: tuple( - (r[c] is None, r[c]) for c in sort_columns - ) - ) - elif record_id_column is not None: - members.sort(key=lambda r: r[record_id_column] or b"") - - grouped_rows.append({ - **dict(zip(self.by, key)), - **{c: [m[c] for m in members] for c in member_columns}, - **{ - c: arrow_utils.fold_system_tag_values( - c, [m[c] for m in members] - ) - for c in system_tag_columns - }, - }) - - input_fields = {f.name: f for f in table.schema} - grouped_schema = pa.schema([ - pa.field(c, pa.list_(input_fields[c].type), nullable=False) - if c in member_columns - else input_fields[c] - for c in table.column_names - ]) - grouped_table = pa.Table.from_pylist(grouped_rows, schema=grouped_schema) - - n_char = self.orcapod_config.hashing.system_tag_n_char - grouped_table = arrow_utils.append_to_system_tags( - grouped_table, stream.pipeline_hash().to_hex(n_char) - ) - - return ArrowTableStream( - grouped_table, - tag_columns=self.by, - data_context=stream.data_context, - ) - - # ------------------------------------------------------------------ - # Schema prediction - # ------------------------------------------------------------------ - - def unary_output_schema( - self, - stream: StreamProtocol, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Predict the grouped output schemas without grouping. - - Args: - stream: The upstream stream. - columns: Column inclusion config. - all_info: Include all info columns. - - Returns: - A ``(tag_schema, data_schema)`` tuple. Group keys stay scalar in - the tag schema; promoted non-key tags and list-wrapped data columns - land in the data schema. - """ - column_config = ColumnConfig.handle_config(columns, all_info=all_info) - tag_types, data_types = stream.output_schema( - columns=columns, all_info=all_info - ) - n_char = self.orcapod_config.hashing.system_tag_n_char - suffix = stream.pipeline_hash().to_hex(n_char) - - out_tag_types: dict[str, Any] = {} - out_data_types: dict[str, Any] = {} - - for name, col_type in tag_types.items(): - if name.startswith(constants.SYSTEM_TAG_PREFIX): - out_tag_types[ - f"{name}{constants.BLOCK_SEPARATOR}{suffix}" - ] = col_type - elif name in self.by: - out_tag_types[name] = col_type - else: - # Promoted to a list-valued data column. - out_data_types[name] = list[col_type] - if column_config.source: - # Promoted columns carry no provenance token; the stream - # fills in a scalar null. - out_data_types[f"{constants.SOURCE_PREFIX}{name}"] = str - - for name, col_type in data_types.items(): - out_data_types[name] = list[col_type] - - return Schema(out_tag_types), Schema(out_data_types) -``` - -- [ ] **Step 4: Export it** - -In `src/orcapod/core/operators/__init__.py`, add the import alphabetically after `from .filters import PolarsFilter`: - -```python -from .group_by import GroupBy -``` - -and add `"GroupBy",` to `__all__` after `"Batch",`. - -- [ ] **Step 5: Run the tests to verify they pass** - -```bash -uv run pytest tests/test_core/operators/test_group_by.py -v -``` - -Expected: all PASS. If `test_row_order_does_not_affect_output` fails, the sort key is wrong — check that `sort_columns` excludes the `by` columns and that `members.sort` runs before the dict comprehension. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/core/operators/group_by.py src/orcapod/core/operators/__init__.py tests/test_core/operators/test_group_by.py -git commit -m "feat(operators): add GroupBy for many-to-one tag-keyed reduction (NPIPE-204) - -Every other operator preserves one row per tag; GroupBy collapses N rows -sharing a tag tuple into one packet with list-valued members, which is what -lets a downstream pod receive a whole recording session at once. - -Group keys stay scalar tags; non-key tags are promoted to list-valued data -columns rather than dropped, so consumers can tell which member each element -came from. Members sort by non-key tag values so the hashed lists are stable -across runs. - -NPIPE-204" -``` - ---- - -## Task 6: Schema-mirroring test - -**Files:** -- Test: `tests/test_core/operators/test_group_by.py` - -`unary_output_schema` predicting something the table does not match is the classic operator bug — it surfaces far downstream, as a DB write failure rather than a schema error. - -- [ ] **Step 1: Write the failing test** - -Append to `tests/test_core/operators/test_group_by.py`: - -```python -class TestGroupBySchemaMirror: - """unary_output_schema must match what the produced stream reports.""" - - @pytest.mark.parametrize( - "config", - [ - {}, - {"source": True}, - {"system_tags": True}, - {"source": True, "system_tags": True}, - ], - ids=["bare", "source", "system_tags", "source+system_tags"], - ) - def test_predicted_schema_matches_produced_stream(self, session_source, config): - op = GroupBy(by=["subject", "date"]) - - predicted_tags, predicted_data = op.unary_output_schema( - session_source, columns=config - ) - produced = op.unary_static_process(session_source) - actual_tags, actual_data = produced.output_schema(columns=config) - - assert dict(predicted_tags) == dict(actual_tags) - assert dict(predicted_data) == dict(actual_data) -``` - -**Two comparisons that look right and are not.** Both were tried during review and both silently pass: - -- **Against `op.process(...)`** — `DynamicPodStream.output_schema` (`static_output_pod.py:321-332`) delegates straight back to the pod, so this compares the prediction against itself and can never fail. -- **Against `as_table(columns=config).column_names`** — `output_schema` and `as_table` legitimately disagree. `ArrowTableStream.output_schema` (`arrow_table_stream.py:183-204`) returns `self._data_schema` unconditionally, so `columns.source` is a documented no-op there (`source_node.py:178-180` states this). `_source_*`, `_content_hash`, and `_context_key` appear in the table but never in the schema. The same gap exists on a plain un-batched `ArrowTableSource`, so it is a property of the schema layer, not of any operator. - -Compare the prediction against the materialized `ArrowTableStream` from `unary_static_process`. That is the contract that actually binds. - -- [ ] **Step 2: Run the test** - -```bash -uv run pytest tests/test_core/operators/test_group_by.py::TestGroupBySchemaMirror -v -``` - -Expected: PASS if Task 5 was implemented correctly. If a case fails, the assertion message names the exact mismatched columns — fix `unary_output_schema` to match the table, not the other way round. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_core/operators/test_group_by.py -git commit -m "test(group_by): assert output_schema mirrors as_table (NPIPE-204) - -NPIPE-204" -``` - ---- - -## Task 7: Registration — serialization and fluent API - -**Files:** -- Modify: `src/orcapod/pipeline/serialization.py:160-192` -- Modify: `src/orcapod/core/streams/base.py:135-149` -- Test: `tests/test_core/operators/test_group_by.py` - -Without the registry entry, a pipeline containing a `GroupBy` cannot be deserialized. - -- [ ] **Step 1: Write the failing test** - -Append to `tests/test_core/operators/test_group_by.py`: - -```python -class TestGroupByRegistration: - def test_in_operator_registry(self): - from orcapod.pipeline.serialization import _build_operator_registry - - assert _build_operator_registry()["GroupBy"] is GroupBy - - def test_stream_fluent_method(self, session_source): - out = session_source.group_by(["subject", "date"]) - assert len(out.as_table()) == 2 - - -class TestGroupByAsyncIsBarrier: - """GroupBy must NOT override async_execute. - - ``UnaryOperator.async_execute`` (``core/operators/base.py:71``) already - collects the full input before calling ``static_process``, which is exactly - the barrier GroupBy needs: no group can be emitted before the input channel - closes, because any row not yet seen could belong to a group already - started. Adding an override would duplicate that logic and risk drifting - from it. - """ - - def test_does_not_override_async_execute(self): - from orcapod.core.operators.base import UnaryOperator - - assert "async_execute" not in GroupBy.__dict__ - assert GroupBy.async_execute is UnaryOperator.async_execute -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_core/operators/test_group_by.py::TestGroupByRegistration -v -``` - -Expected: `test_in_operator_registry` FAILS with `KeyError: 'GroupBy'`; `test_stream_fluent_method` FAILS with `AttributeError`. - -- [ ] **Step 3: Add the registry entry** - -In `src/orcapod/pipeline/serialization.py`, inside `_build_operator_registry`, add `GroupBy` to the import list (alphabetically, after `DropTagColumns`): - -```python - from orcapod.core.operators import ( - Batch, - DropDataColumns, - DropTagColumns, - GroupBy, - Join, - MapData, - MapTags, - MergeJoin, - PolarsFilter, - SelectDataColumns, - SelectTagColumns, - SemiJoin, - ) -``` - -and add the entry to the returned dict, after `"Batch": Batch,`: - -```python - "GroupBy": GroupBy, -``` - -- [ ] **Step 4: Add the fluent method** - -In `src/orcapod/core/streams/base.py`, immediately after the existing `batch` method (ends line 149), add: - -```python - def group_by( - self, - by: Collection[str], - label: str | None = None, - ) -> StreamBase: - """Reduce rows sharing a tag tuple into one packet per group. - - Group-key columns stay scalar tags; every other column becomes - list-valued. See ``orcapod.core.operators.GroupBy``. - - Args: - by: Tag column names to group on. - label: Optional node label for the pipeline graph. - - Returns: - A stream with one row per distinct group-key tuple. - """ - from orcapod.core.operators import GroupBy - - return GroupBy(by=by)(self, label=label) -``` - -`Collection` is already imported at the top of `base.py`. Verify with `grep -n "from collections.abc" src/orcapod/core/streams/base.py` and add it to that import if absent. - -- [ ] **Step 5: Run the tests to verify they pass** - -```bash -uv run pytest tests/test_core/operators/test_group_by.py -v -uv run pytest tests/test_pipeline/test_serialization.py -q -``` - -Expected: all PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/pipeline/serialization.py src/orcapod/core/streams/base.py tests/test_core/operators/test_group_by.py -git commit -m "feat(operators): register GroupBy in serialization and stream API (NPIPE-204) - -Without the registry entry a pipeline containing a GroupBy cannot be -deserialized. - -NPIPE-204" -``` - ---- - -## Task 8: Job-level tests - -**Files:** -- Create: `tests/test_pipeline/test_aggregation_job.py` - -This is the gap that let the provenance crash ship. Every existing `Batch` test stops at `op.process(stream)` + `as_table()`; the crash only happens inside `job.run()`, in `StaticOutputOperatorPod._materialize_to_stream`. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_pipeline/test_aggregation_job.py`: - -```python -"""Job-level tests for aggregating operators (NPIPE-204). - -Operator-level tests (`op.process(stream)` then `as_table()`) never reach -`StaticOutputOperatorPod._materialize_to_stream`, which is where list-valued -provenance used to crash. These run the full `job.run()` path against a real -Delta Lake store. -""" - -from __future__ import annotations - -from pathlib import Path - -import pyarrow as pa -import pytest - -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod -from orcapod.core.operators import Batch, GroupBy, MergeJoin -from orcapod.core.sources import ArrowTableSource -from orcapod.databases import DeltaTableDatabase -from orcapod.pipeline import PipelineJob - - -@pytest.fixture -def store(tmp_path: Path) -> DeltaTableDatabase: - return DeltaTableDatabase(str(tmp_path / "store")) - - -@pytest.fixture -def session_source_factory(): - """Build a 2-group source; `paths` lets a test mutate one group's data.""" - - def _make(paths: list[str] | None = None) -> ArrowTableSource: - table = pa.table({ - "subject": ["G", "G", "G", "G"], - "date": ["d1", "d1", "d2", "d2"], - "probe": [0, 1, 0, 1], - "path": paths or ["a", "b", "c", "d"], - }) - return ArrowTableSource( - table, - tag_columns=["subject", "date", "probe"], - infer_nullable=True, - ) - - return _make - - -class _CountingFunction: - """Records every invocation so memoization can be asserted on. - - `PythonDataFunction` binds data columns to parameter names via - `inspect.signature`, which resolves a callable instance to `__call__`. - The parameter must therefore be named after the data column — hence the - separate `_CountingV` below for MergeJoin's merged `v` column. - """ - - def __init__(self) -> None: - self.calls: list[list[str]] = [] - - def __call__(self, path: list[str]) -> int: - self.calls.append(list(path)) - return len(path) - - -class _CountingV: - """Same as `_CountingFunction`, bound to a column named `v`.""" - - def __init__(self) -> None: - self.calls: list[list[str]] = [] - - def __call__(self, v: list[str]) -> int: - self.calls.append(list(v)) - return len(v) - - -def _run(store, source, operator, fn, name): - pod = FunctionPod(PythonDataFunction(fn, output_keys="n", function_name="count")) - job = PipelineJob(name=name, store=store) - with job: - pod(operator(source, label="agg"), label="counter") - return job.run() - - -class TestGroupByInJob: - def test_group_by_completes(self, store, session_source_factory): - fn = _CountingFunction() - _run(store, session_source_factory(), GroupBy(by=["subject", "date"]), fn, "gb") - assert len(fn.calls) == 2 - assert sorted(fn.calls) == [["a", "b"], ["c", "d"]] - - def test_batch_completes(self, store, session_source_factory): - """The provenance fix is independent of grouping.""" - fn = _CountingFunction() - _run(store, session_source_factory(), Batch(batch_size=2), fn, "b") - assert len(fn.calls) == 2 - - -class TestGroupByMemoization: - def test_identical_runs_hit_cache(self, store, session_source_factory): - fn = _CountingFunction() - _run(store, session_source_factory(), GroupBy(by=["subject", "date"]), fn, "m") - assert len(fn.calls) == 2 - - fn2 = _CountingFunction() - _run(store, session_source_factory(), GroupBy(by=["subject", "date"]), fn2, "m") - assert fn2.calls == [], "second identical run must not recompute" - - def test_changed_member_invalidates_only_its_group( - self, store, session_source_factory - ): - """Two groups; change one member of the first only. - - With a single group this assertion would be vacuous -- it must show - that the untouched group stays cached. - """ - fn = _CountingFunction() - _run(store, session_source_factory(), GroupBy(by=["subject", "date"]), fn, "i") - assert len(fn.calls) == 2 - - fn2 = _CountingFunction() - changed = session_source_factory(["a", "B_CHANGED", "c", "d"]) - _run(store, changed, GroupBy(by=["subject", "date"]), fn2, "i") - - assert fn2.calls == [["a", "B_CHANGED"]], ( - "only the changed group should recompute; " - f"got {fn2.calls}" - ) - - -class TestMergeJoinRegression: - def test_merge_join_completes_in_job(self, store): - """MergeJoin carries source columns as parallel lists (merge_join.py:262). - - It crashed with the same ArrowTypeError before the Data fix. - """ - left = ArrowTableSource( - pa.table({"id": ["a", "b"], "v": ["l1", "l2"]}), - tag_columns=["id"], - infer_nullable=True, - ) - right = ArrowTableSource( - pa.table({"id": ["a", "b"], "v": ["r1", "r2"]}), - tag_columns=["id"], - infer_nullable=True, - ) - - fn = _CountingV() - pod = FunctionPod( - PythonDataFunction(fn, output_keys="n", function_name="count_v") - ) - job = PipelineJob(name="mj", store=store) - with job: - pod(MergeJoin()(left, right, label="mj"), label="counter") - job.run() - - assert len(fn.calls) == 2 - # MergeJoin merges colliding `v` columns into a sorted 2-element list. - assert sorted(fn.calls) == [["l1", "r1"], ["l2", "r2"]] -``` - -- [ ] **Step 2: Run test to verify it fails, and how** - -```bash -uv run pytest tests/test_pipeline/test_aggregation_job.py -v -``` - -Expected before Tasks 1–7: `ArrowTypeError: Expected bytes, got a 'list' object`. After Tasks 1–7: these should pass. If a memoization test fails, that is a real finding — report the actual `fn2.calls` value rather than relaxing the assertion. - -- [ ] **Step 3: Fix any failures found** - -If `test_changed_member_invalidates_only_its_group` shows *both* groups recomputing, the likely cause is that `fold_system_tag_values` is being fed members in a different order between runs — verify the sort in `unary_static_process` runs before the fold. If *neither* recomputes, the input data hash is not reaching the record preimage; inspect `_build_record_id_preimage` output for both runs. - -- [ ] **Step 4: Run the full suite** - -```bash -uv run pytest tests/ -q -``` - -Expected: no failures. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_pipeline/test_aggregation_job.py -git commit -m "test(pipeline): job-level coverage for GroupBy, Batch, MergeJoin (NPIPE-204) - -Every existing aggregating-operator test stopped at op.process() + as_table(), -which never reaches _materialize_to_stream -- the reason the list-valued -provenance crash shipped. These run the full job.run() path against a Delta -Lake store and assert memoization holds across identical runs and invalidates -only the group whose member changed. - -NPIPE-204" -``` - ---- - -## Task 9: Documentation - -**Files:** -- Modify: `CLAUDE.md` -- Modify: `.zed/rules` -- Modify: `DESIGN_ISSUES.md` - -Per `CLAUDE.md`, agent instructions must be updated in **both** `CLAUDE.md` and `.zed/rules`. - -- [ ] **Step 1: Update the project layout tree** - -In `CLAUDE.md`, in the `src/orcapod/core/operators/` block, add after the `batch.py` line: - -``` -│ ├── group_by.py # GroupBy (many→one reduction keyed on tag values) -``` - -- [ ] **Step 2: Update the operator / function pod boundary table** - -`GroupBy` synthesizes no new values but does change row count. Add a note under that table in `CLAUDE.md`: - -```markdown -`GroupBy` is the only operator that changes row count in a many→one direction: it -reduces N rows sharing a tag tuple to one row with list-valued members. It still -synthesizes no new data values — every emitted element came from an input row. -``` - -- [ ] **Step 3: Update the system tag evolution rules** - -In `CLAUDE.md`, the "System tag evolution rules" section currently has three rules, and rule 3 describes `Batch` as purely type-evolving (`str` → `list[str]`). That is now wrong. Replace rule 3 with: - -```markdown -3. **Reducing** — many→one ops (`Batch`, `GroupBy`). User tag and data columns become - `list[T]`; source-info columns become `list[str]`, one element per member. System tag - columns must stay **scalar** (record identity hashes them directly), so they fold to a - deterministic digest via `arrow_utils.fold_system_tag_values` and their column name gains - `::{pipeline_hash}`. The fold is SHA-based and stable across processes — never use - `hash()` there, since the digest becomes a cache key. -``` - -- [ ] **Step 4: Add the `Important implementation details` entries** - -In `CLAUDE.md`, append to that list: - -```markdown -- `GroupBy` requires every column in `by` to be a tag column; raises `InputValidationError` - otherwise. Members are sorted by non-group-key tag values (falling back to system - `record_id`) so the hashed lists are stable across runs. -- `Data` source-info values may be `str`, `None`, or `list[...]`. Types are derived from the - value; `None` maps to `large_string`. -``` - -- [ ] **Step 5: Mirror every change into `.zed/rules`** - -`.zed/rules` carries the same instructions for Zed AI and must stay in sync — this is -required by `CLAUDE.md` itself. Apply Steps 1–4 verbatim to `.zed/rules`, then confirm the -two files agree on this content: - -```bash -diff CLAUDE.md .zed/rules -``` - -Expected: no differences in any section touched by Steps 1–4. If the files already diverge -elsewhere for unrelated reasons, leave those differences alone — only verify that every -line you added appears in both. - -- [ ] **Step 6: Resolve the `tag_data.py` half of U1** - -In `DESIGN_ISSUES.md`, update the U1 status line: - -```markdown -**Status:** resolved (`tag_data.py` half), open (`arrow_utils.py` half) -``` - -and append to the `**Fix (NPIPE-204):**` paragraph: - -```markdown -**Fix:** landed in NPIPE-204. `_source_info_arrow_type` / `_source_info_python_type` in -`core/datagrams/tag_data.py` derive the Arrow and Python types from the stored value. -The dead `polars_data_utils.add_source_info` was deleted. `add_source_info_to_table()` -in `arrow_utils.py` is untouched and remains open. -``` - -- [ ] **Step 7: Verify docs are consistent** - -```bash -uv run pytest tests/ -q -grep -n "group_by.py" CLAUDE.md .zed/rules -grep -n "GroupBy" CLAUDE.md .zed/rules -``` - -Expected: suite passes; both files mention `group_by.py` and `GroupBy`. - -- [ ] **Step 8: Commit** - -```bash -git add CLAUDE.md .zed/rules DESIGN_ISSUES.md -git commit -m "docs: document GroupBy and the reducing system-tag rule (NPIPE-204) - -Rule 3 described Batch as purely type-evolving, which was only ever true of -user tag and data columns -- system tags fold to a scalar and extend their -name. Renames the rule to 'reducing' and covers both operators. - -Resolves the tag_data.py half of DESIGN_ISSUES U1. -NPIPE-204" -``` - ---- - -## Task 10: Final verification and PR - -**Files:** none - -- [ ] **Step 1: Full suite** - -```bash -uv run pytest tests/ -q -``` - -Expected: all pass. Record the exact pass/fail counts — do not claim success without reading the output. - -- [ ] **Step 2: Confirm both original crashes are gone** - -```bash -uv run pytest tests/test_pipeline/test_aggregation_job.py -v -``` - -Expected: `TestGroupByInJob`, `TestGroupByMemoization`, and `TestMergeJoinRegression` all pass. - -- [ ] **Step 3: Review the whole diff** - -```bash -git diff main...HEAD --stat -git diff main...HEAD -``` - -Check: no debug prints, no `sys.modules` manipulation, no backward-compat shims (all forbidden by `CLAUDE.md`), Google-style docstrings with no ReST roles. - -- [ ] **Step 4: Push and open the PR** - -```bash -git push -u origin arnoldb/npipe-204-orcapod-add-tag-based-grouping-to-batch-manyone-reduction -``` - -PR body must include `Fixes NPIPE-204` so Linear's GitHub integration links it, and must call out the API deviation: - -```markdown -## Summary - -Adds a `GroupBy` operator for many→one reduction keyed on tag values, and fixes the -`Data` datagram so list-valued provenance no longer crashes `job.run()`. - -**API note:** NPIPE-204 specifies `Batch(group_by=[...])`. This PR adds a separate -`GroupBy(by=[...])` instead, so each operator keeps one output contract and `Batch`'s -streaming `async_execute` path stays untouched. See -`superpowers/specs/2026-08-07-npipe-204-batch-group-by-design.md`. - -The provenance fix also repairs `MergeJoin`, which produced list-valued `_source_*` -columns (`merge_join.py:262`) and was silently broken inside `job.run()`. - -Fixes NPIPE-204 - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -``` - -- [ ] **Step 5: Update the Linear issue** - -NPIPE-204's "What to build" section and the `common_clock_op` wiring example both still say `Batch(group_by=...)`. Update them to `GroupBy(by=...)` so the downstream orcapod-sync-and-qc change is written against the real API. - -**Confirm with the user before editing the Linear issue** — it is an outward-facing change to a shared record. - ---- - -## Downstream (not part of this plan) - -The rev bump must land in **both** orcapod-sync-and-qc and orcapod-spikesorting together — they are deliberately kept on the same orcapod rev because they target the same Ray cluster. `common_clock_op` wiring becomes: - -```python -grouped = op.operators.GroupBy(by=["subject", "date"]).process(sync_out) -common_clock_op.pod(grouped) -``` diff --git a/superpowers/plans/2026-08-11-itl-173-list-set-logical-types.md b/superpowers/plans/2026-08-11-itl-173-list-set-logical-types.md deleted file mode 100644 index 9fd8a0acb..000000000 --- a/superpowers/plans/2026-08-11-itl-173-list-set-logical-types.md +++ /dev/null @@ -1,1745 +0,0 @@ -# ITL-173: List and Set Logical Types Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix `register_python_class(list[T])` / `set[T]` where T is a registered logical type — currently raises `ValueError` — by wrapping the entire container as a top-level Arrow extension type that embeds element reconstruction metadata. - -**Architecture:** A new `ListLogicalType` class wraps `list[T]`/`set[T]` (where T resolves to a `pa.ExtensionType`) as a top-level Arrow extension over `large_list()`. Metadata JSON at the field level encodes the element extension name and its own metadata, enabling recursive reconstruction on read. Two guard removals in `UniversalTypeConverter` allow `GenericAlias` keys to be looked up in the registry. A shared helper `_make_or_get_list_logical_type` is called from both `_register_python_class_impl` and `_convert_python_to_arrow` for idempotent creation. - -**Tech Stack:** PyArrow, Polars, Python `types.GenericAlias`, `typing.get_origin`/`get_args`, existing `make_arrow_extension_type`/`make_polars_extension_type` helpers, `BaseLogicalType`. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `src/orcapod/extension_types/list_logical_type_factory.py` | **Create** | `ListLogicalType`, `ListLogicalTypeFactory`, `LIST_CATEGORY`, `SET_CATEGORY` | -| `src/orcapod/extension_types/registry.py` | **Modify** line 297 | Guard `__qualname__` against `GenericAlias` | -| `src/orcapod/extension_types/protocols.py` | **Modify** | Add `arrow_type_to_python_type` to `TypeConverterProtocol` | -| `src/orcapod/semantic_types/universal_converter.py` | **Modify** 6 sites | Remove `isinstance(python_type, type)` guard ×2; replace `ValueError` raises ×2; update `_convert_python_to_arrow` list/set branches ×2; add `_make_or_get_list_logical_type` helper | -| `src/orcapod/extension_types/__init__.py` | **Modify** | Export new symbols | -| `src/orcapod/contexts/data/v0.1.json` | **Modify** | Register factory under `"list"` and `"set"` categories | -| `DESIGN_ISSUES.md` | **Modify** | Update ET2 status to `resolved` | -| `tests/test_extension_types/test_list_logical_type.py` | **Create** | Unit tests for `ListLogicalType` and `ListLogicalTypeFactory` | -| `tests/test_extension_types/test_roundtrips.py` | **Modify** | Add 9 integration round-trip tests | - ---- - -## Task 1: Fix `registry.py` — guard `__qualname__` for `GenericAlias` - -**Files:** -- Modify: `src/orcapod/extension_types/registry.py:297` -- Test: `tests/test_extension_types/test_registry.py` - -`register_logical_type` uses `py_type.__qualname__` in the conflict-error message. `GenericAlias` (e.g. `list[uuid.UUID]`) has no `__qualname__`. The fix uses `getattr` with a fallback. - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_extension_types/test_registry.py`: - -```python -def test_register_logical_type_conflict_error_uses_repr_for_generic_alias(): - """Conflict error message for GenericAlias python_type must not raise AttributeError.""" - import uuid - import pyarrow as pa - from orcapod.extension_types.registry import LogicalTypeRegistry, make_arrow_extension_type - from orcapod.extension_types.base_logical_type import BaseLogicalType - - class _FakeListLT(BaseLogicalType): - logical_type_name = "list[orcapod.uuid]" - python_type = list[uuid.UUID] - - def get_arrow_extension_type(self): - ext_cls = make_arrow_extension_type( - "list[orcapod.uuid]", pa.large_list(pa.large_binary()) - ) - return ext_cls() - - def get_polars_extension_type(self): - from orcapod.extension_types.registry import make_polars_extension_type - ext_cls = make_polars_extension_type( - "list[orcapod.uuid]", pa.large_list(pa.large_binary()) - ) - return ext_cls() - - def python_to_storage(self, value, converter): - return value - - def storage_to_python(self, storage_value, converter): - return storage_value - - class _FakeListLT2(_FakeListLT): - """Different instance, same keys — should raise ValueError, not AttributeError.""" - - registry = LogicalTypeRegistry() - lt1 = _FakeListLT() - lt2 = _FakeListLT2() - registry.register_logical_type(lt1) - - with pytest.raises(ValueError, match="python_type"): - registry.register_logical_type(lt2) -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_extension_types/test_registry.py::test_register_logical_type_conflict_error_uses_repr_for_generic_alias -v -``` - -Expected: `FAILED` with `AttributeError: 'types.GenericAlias' object has no attribute '__qualname__'` - -- [ ] **Step 3: Apply the fix** - -In `src/orcapod/extension_types/registry.py`, find line 297 (inside `register_logical_type`, the `for existing, label, key in [...]` block): - -```python -# Before -(existing_by_python, "python_type", py_type.__qualname__), - -# After -(existing_by_python, "python_type", getattr(py_type, "__qualname__", repr(py_type))), -``` - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_registry.py::test_register_logical_type_conflict_error_uses_repr_for_generic_alias -v -``` - -Expected: `PASSED` - -- [ ] **Step 5: Run full registry tests to confirm no regression** - -```bash -uv run pytest tests/test_extension_types/test_registry.py -v -``` - -Expected: all `PASSED` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/registry.py tests/test_extension_types/test_registry.py -git commit -m "fix(registry): guard __qualname__ against GenericAlias in conflict error message" -``` - ---- - -## Task 2: Add `arrow_type_to_python_type` to `TypeConverterProtocol` - -**Files:** -- Modify: `src/orcapod/extension_types/protocols.py` -- Test: `tests/test_extension_types/test_protocols.py` - -`ListLogicalTypeFactory.reconstruct_from_arrow` needs `converter.arrow_type_to_python_type(element_ext)` to recover the element's Python type after registering it. This method already exists on `UniversalTypeConverter` but is not in `TypeConverterProtocol`. - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_extension_types/test_protocols.py`: - -```python -def test_type_converter_protocol_has_arrow_type_to_python_type(): - """TypeConverterProtocol must declare arrow_type_to_python_type.""" - import inspect - from orcapod.extension_types.protocols import TypeConverterProtocol - assert "arrow_type_to_python_type" in TypeConverterProtocol.__protocol_attrs__ -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py::test_type_converter_protocol_has_arrow_type_to_python_type -v -``` - -Expected: `FAILED` with `AssertionError` - -- [ ] **Step 3: Add the method to the protocol** - -In `src/orcapod/extension_types/protocols.py`, add after `register_arrow_extension`: - -```python - def arrow_type_to_python_type(self, arrow_type: "pa.DataType") -> "DataType": - """Convert an Arrow type to its Python type hint. - - Used by ``ListLogicalTypeFactory.reconstruct_from_arrow`` to recover - the element Python type after registering the element extension type. - - Args: - arrow_type: An Arrow type (may be a ``pa.ExtensionType``). - - Returns: - The Python type hint corresponding to ``arrow_type``. - """ - ... -``` - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py::test_type_converter_protocol_has_arrow_type_to_python_type -v -``` - -Expected: `PASSED` - -- [ ] **Step 5: Run all protocol tests** - -```bash -uv run pytest tests/test_extension_types/test_protocols.py -v -``` - -Expected: all `PASSED` - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/protocols.py tests/test_extension_types/test_protocols.py -git commit -m "feat(protocols): add arrow_type_to_python_type to TypeConverterProtocol" -``` - ---- - -## Task 3: Create `list_logical_type_factory.py` — `ListLogicalType` - -**Files:** -- Create: `src/orcapod/extension_types/list_logical_type_factory.py` -- Create: `tests/test_extension_types/test_list_logical_type.py` - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_extension_types/test_list_logical_type.py`: - -```python -"""Tests for ListLogicalType and ListLogicalTypeFactory.""" -from __future__ import annotations - -import uuid as uuid_module - -import pyarrow as pa -import pytest - -# ── Helpers ─────────────────────────────────────────────────────────────────── - - -def _uuid_ext_type() -> pa.ExtensionType: - """Return the registered orcapod.uuid extension type.""" - from orcapod.extension_types.builtin_logical_types import LogicalUUID - return LogicalUUID().get_arrow_extension_type() - - -class _StubConverter: - """Minimal converter stub delegating UUID and list[UUID] conversions.""" - - def python_to_storage(self, value, annotation): - if annotation is uuid_module.UUID: - return value.bytes - if hasattr(annotation, "__origin__") and annotation.__origin__ is list: - import typing - args = typing.get_args(annotation) - return [self.python_to_storage(item, args[0]) for item in value] - return value - - def storage_to_python(self, storage_value, annotation): - if annotation is uuid_module.UUID: - return uuid_module.UUID(bytes=bytes(storage_value)) - if hasattr(annotation, "__origin__") and annotation.__origin__ is list: - import typing - args = typing.get_args(annotation) - return [self.storage_to_python(item, args[0]) for item in storage_value] - return storage_value - - def register_python_class(self, annotation): - if annotation is uuid_module.UUID: - return _uuid_ext_type() - if annotation is str: - return pa.large_string() - if annotation is int: - return pa.int64() - raise ValueError(f"Unsupported annotation: {annotation}") - - def register_arrow_extension(self, ext_name, metadata_bytes, storage_type): - if ext_name == "orcapod.uuid": - return _uuid_ext_type() - raise ValueError(f"Unknown extension: {ext_name}") - - def arrow_type_to_python_type(self, arrow_type): - if hasattr(arrow_type, "extension_name"): - if arrow_type.extension_name == "orcapod.uuid": - return uuid_module.UUID - return type(None) - - -# ── ListLogicalType unit tests ──────────────────────────────────────────────── - - -def test_list_logical_type_importable(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - assert ListLogicalType is not None - - -def test_list_logical_type_name(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - assert lt.logical_type_name == "list[orcapod.uuid]" - - -def test_set_logical_type_name(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=True) - assert lt.logical_type_name == "set[orcapod.uuid]" - - -def test_list_logical_type_python_type(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - assert lt.python_type == list[uuid_module.UUID] - - -def test_set_logical_type_python_type(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=True) - assert lt.python_type == set[uuid_module.UUID] - - -def test_list_logical_type_arrow_extension_type_name(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - ext = lt.get_arrow_extension_type() - assert hasattr(ext, "extension_name") - assert ext.extension_name == "list[orcapod.uuid]" - - -def test_list_logical_type_storage_is_large_list_of_large_binary(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - ext = lt.get_arrow_extension_type() - assert pa.types.is_large_list(ext.storage_type) - assert ext.storage_type.value_type == pa.large_binary() - - -def test_list_logical_type_storage_is_et1_safe(): - """Value type of list storage must NOT be a pa.ExtensionType (ET1 invariant).""" - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - ext = lt.get_arrow_extension_type() - assert not isinstance(ext.storage_type.value_type, pa.ExtensionType), ( - "ET1 violation: list value type must not be an ExtensionType" - ) - - -def test_list_logical_type_arrow_extension_type_cached(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - assert lt.get_arrow_extension_type() is lt.get_arrow_extension_type() - - -def test_list_logical_type_index_element(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - assert lt.index_element() == uuid_module.UUID - - -def test_list_logical_type_python_to_storage(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - u2 = uuid_module.UUID("87654321-4321-8765-4321-876543218765") - result = lt.python_to_storage([u1, u2], _StubConverter()) - assert result == [u1.bytes, u2.bytes] - - -def test_list_logical_type_python_to_storage_none(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - assert lt.python_to_storage(None, _StubConverter()) == [] - - -def test_list_logical_type_storage_to_python(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - result = lt.storage_to_python([u1.bytes], _StubConverter()) - assert result == [u1] - assert isinstance(result, list) - - -def test_set_logical_type_storage_to_python_returns_set(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=True) - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - result = lt.storage_to_python([u1.bytes], _StubConverter()) - assert isinstance(result, set) - assert result == {u1} - - -def test_list_logical_type_metadata_contains_element_ext_name(): - """Extension metadata must contain element_ext_name for reconstruction.""" - import json - from orcapod.extension_types.list_logical_type_factory import ListLogicalType, LIST_CATEGORY - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - ext = lt.get_arrow_extension_type() - meta = json.loads(ext.__arrow_ext_serialize__().decode("utf-8")) - assert meta["category"] == LIST_CATEGORY - assert meta["element_ext_name"] == "orcapod.uuid" - assert "element_ext_metadata" in meta - - -def test_list_logical_type_protocol_conformance(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - from orcapod.extension_types.protocols import LogicalTypeProtocol - lt = ListLogicalType(uuid_module.UUID, _uuid_ext_type(), is_set=False) - assert isinstance(lt, LogicalTypeProtocol) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_list_logical_type.py -v 2>&1 | head -30 -``` - -Expected: multiple `FAILED` or `ERROR` with `ModuleNotFoundError: No module named 'orcapod.extension_types.list_logical_type_factory'` - -- [ ] **Step 3: Create `list_logical_type_factory.py` with `ListLogicalType`** - -Create `src/orcapod/extension_types/list_logical_type_factory.py`: - -```python -"""ListLogicalType and ListLogicalTypeFactory. - -Provides ``ListLogicalType`` — a logical type that wraps ``list[T]`` or ``set[T]`` -(where ``T`` is itself a registered logical type) as a top-level Arrow extension -type with storage ``large_list()``. - -This solves ET2 (see ``DESIGN_ISSUES.md``): Arrow cannot preserve extension-type -metadata inside list value fields (ET1), but metadata at the outermost field level -IS preserved through Parquet and Delta. By wrapping the whole list as the extension -type, the element type information survives a Parquet round-trip with a fresh -converter. - -Write path: triggered explicitly in ``UniversalTypeConverter._register_python_class_impl`` -and ``_convert_python_to_arrow`` when ``list[T]``/``set[T]`` is encountered and ``T`` -resolves to a ``pa.ExtensionType``. - -Read path (``ListLogicalTypeFactory.reconstruct_from_arrow``): dispatched by -``register_arrow_extension`` when category is ``"list"`` or ``"set"``. Recursively -calls ``converter.register_arrow_extension`` for the element type. - -Category tags: ``LIST_CATEGORY = "list"``, ``SET_CATEGORY = "set"`` -""" - -from __future__ import annotations - -import json -import logging -from typing import TYPE_CHECKING, Any - -from orcapod.extension_types.base_logical_type import BaseLogicalType -from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from orcapod.extension_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) - -#: Category tag embedded in Arrow extension metadata for ``list[T]`` types. -LIST_CATEGORY = "list" - -#: Category tag embedded in Arrow extension metadata for ``set[T]`` types. -SET_CATEGORY = "set" - - -class ListLogicalType(BaseLogicalType): - """Logical type for ``list[T]`` or ``set[T]`` where ``T`` is a registered logical type. - - Wraps the entire container as a top-level Arrow extension type with storage - ``large_list()``. Extension-type metadata is only at the outermost - field level (storage-safe invariant), encoding enough information to reconstruct - the element type recursively on read. - - ``python_type`` returns the full generic alias (e.g. ``list[uuid.UUID]``) so that - ``arrow_schema_to_python_schema`` produces a round-trippable schema. - - Args: - element_python_type: The Python type for list elements - (e.g. ``uuid.UUID``, ``list[uuid.UUID]`` for nested lists). - element_ext_type: The ``pa.ExtensionType`` for the element logical type. - Its ``storage_type`` becomes the list value type; its - ``__arrow_ext_serialize__()`` bytes are embedded in this type's metadata. - is_set: If ``True``, represents ``set[T]`` instead of ``list[T]``. - Storage is identical; ``python_type`` returns ``set[T]`` and - ``storage_to_python`` returns a ``set``. - - Example: - >>> import uuid - >>> uuid_ext = LogicalUUID().get_arrow_extension_type() - >>> lt = ListLogicalType(uuid.UUID, uuid_ext, is_set=False) - >>> lt.logical_type_name - 'list[orcapod.uuid]' - >>> lt.python_type - list[uuid.UUID] - """ - - def __init__( - self, - element_python_type: type, - element_ext_type: "pa.ExtensionType", - *, - is_set: bool = False, - ) -> None: - self._element_python_type = element_python_type - self._element_ext_type = element_ext_type - self._is_set = is_set - - element_ext_name = element_ext_type.extension_name - element_storage = element_ext_type.storage_type - - prefix = "set" if is_set else "list" - self._logical_name = f"{prefix}[{element_ext_name}]" - - # Embed element metadata for recursive reconstruction on read. - # Empty bytes (e.g. UUID has no metadata) → stored as null JSON value. - element_meta_bytes = element_ext_type.__arrow_ext_serialize__() - element_meta_str = element_meta_bytes.decode("utf-8") if element_meta_bytes else None - - category = SET_CATEGORY if is_set else LIST_CATEGORY - _metadata_dict: dict[str, Any] = { - "category": category, - "element_ext_name": element_ext_name, - "element_ext_metadata": element_meta_str, - } - _metadata_bytes = json.dumps(_metadata_dict).encode("utf-8") - - # Storage: large_list() — no nested extension type (ET1 safe). - outer_storage = pa.large_list(element_storage) - - self._arrow_ext_class = make_arrow_extension_type( - self._logical_name, outer_storage, metadata=_metadata_bytes - ) - self._arrow_ext: pa.ExtensionType | None = None - - self._polars_ext_class = make_polars_extension_type( - self._logical_name, outer_storage, metadata=json.dumps(_metadata_dict), - ) - self._polars_ext: pl.BaseExtension | None = None - - @property - def logical_type_name(self) -> str: - """Arrow extension name, e.g. ``list[orcapod.uuid]`` or ``set[orcapod.uuid]``.""" - return self._logical_name - - @property - def python_type(self) -> type: - """Full generic alias for the container type. - - Returns ``list[element_python_type]`` or ``set[element_python_type]``. - The full alias (not bare ``list``) is required so that - ``arrow_schema_to_python_schema`` produces a round-trippable schema. - """ - container = set if self._is_set else list - return container[self._element_python_type] - - def get_arrow_extension_type(self) -> "pa.ExtensionType": - """Return the Arrow extension type (cached). - - Returns: - A ``pa.ExtensionType`` with ``extension_name == logical_type_name`` - and ``storage_type == large_list()``. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> "pl.BaseExtension": - """Return the Polars extension type (cached). - - Returns: - A ``pl.BaseExtension`` registered under ``logical_type_name``. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext - - def index_element(self) -> type: - """Return the Python element type for positional list access. - - Returns: - The ``element_python_type`` passed at construction. - """ - return self._element_python_type - - def python_to_storage(self, value: Any, converter: "TypeConverterProtocol | None") -> list: - """Convert a ``list[T]`` or ``set[T]`` to its Arrow storage representation. - - Delegates per-element conversion to ``converter.python_to_storage``. - - Args: - value: A Python ``list`` or ``set`` of elements of type ``T``. - converter: Active converter for element-level delegation. - - Returns: - A Python list of element storage values (one per input element). - """ - if value is None: - return [] - return [ - converter.python_to_storage(item, self._element_python_type) - for item in value - ] - - def storage_to_python(self, storage_value: Any, converter: "TypeConverterProtocol | None") -> list | set: - """Reconstruct a ``list[T]`` or ``set[T]`` from Arrow storage values. - - Delegates per-element conversion to ``converter.storage_to_python``. - - Args: - storage_value: Sequence of element storage values from an Arrow array. - converter: Active converter for element-level delegation. - - Returns: - A Python ``list`` (or ``set`` when ``is_set=True``) of reconstructed elements. - """ - if storage_value is None: - return set() if self._is_set else [] - elements = [ - converter.storage_to_python(item, self._element_python_type) - for item in storage_value - ] - return set(elements) if self._is_set else elements -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_list_logical_type.py -v -``` - -Expected: all `PASSED` - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/list_logical_type_factory.py tests/test_extension_types/test_list_logical_type.py -git commit -m "feat(extension_types): add ListLogicalType for list[T]/set[T] wrapping" -``` - ---- - -## Task 4: Add `ListLogicalTypeFactory` to `list_logical_type_factory.py` - -**Files:** -- Modify: `src/orcapod/extension_types/list_logical_type_factory.py` -- Modify: `tests/test_extension_types/test_list_logical_type.py` - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/test_extension_types/test_list_logical_type.py`: - -```python -# ── ListLogicalTypeFactory tests ────────────────────────────────────────────── - - -def test_list_logical_type_factory_importable(): - from orcapod.extension_types.list_logical_type_factory import ListLogicalTypeFactory - assert ListLogicalTypeFactory is not None - - -def test_list_logical_type_factory_reconstruct_list_of_uuid(): - """reconstruct_from_arrow produces ListLogicalType(uuid.UUID, …, is_set=False).""" - from orcapod.extension_types.list_logical_type_factory import ( - ListLogicalTypeFactory, - LIST_CATEGORY, - ) - factory = ListLogicalTypeFactory() - storage_type = pa.large_list(pa.large_binary()) - metadata = { - "category": LIST_CATEGORY, - "element_ext_name": "orcapod.uuid", - "element_ext_metadata": None, - } - lt = factory.reconstruct_from_arrow( - "list[orcapod.uuid]", storage_type, metadata, _StubConverter() - ) - assert lt.logical_type_name == "list[orcapod.uuid]" - assert lt.python_type == list[uuid_module.UUID] - - -def test_list_logical_type_factory_reconstruct_set_of_uuid(): - """reconstruct_from_arrow produces ListLogicalType(uuid.UUID, …, is_set=True).""" - from orcapod.extension_types.list_logical_type_factory import ( - ListLogicalTypeFactory, - SET_CATEGORY, - ) - factory = ListLogicalTypeFactory() - storage_type = pa.large_list(pa.large_binary()) - metadata = { - "category": SET_CATEGORY, - "element_ext_name": "orcapod.uuid", - "element_ext_metadata": None, - } - lt = factory.reconstruct_from_arrow( - "set[orcapod.uuid]", storage_type, metadata, _StubConverter() - ) - assert lt.logical_type_name == "set[orcapod.uuid]" - assert lt.python_type == set[uuid_module.UUID] - - -def test_list_logical_type_factory_reconstruct_raises_on_non_list_storage(): - from orcapod.extension_types.list_logical_type_factory import ( - ListLogicalTypeFactory, - LIST_CATEGORY, - ) - factory = ListLogicalTypeFactory() - metadata = {"category": LIST_CATEGORY, "element_ext_name": "orcapod.uuid", "element_ext_metadata": None} - with pytest.raises(ValueError, match="list storage"): - factory.reconstruct_from_arrow( - "list[orcapod.uuid]", pa.large_binary(), metadata, _StubConverter() - ) - - -def test_list_logical_type_factory_reconstruct_raises_on_missing_element_ext_name(): - from orcapod.extension_types.list_logical_type_factory import ( - ListLogicalTypeFactory, - LIST_CATEGORY, - ) - factory = ListLogicalTypeFactory() - metadata = {"category": LIST_CATEGORY} - with pytest.raises(ValueError, match="element_ext_name"): - factory.reconstruct_from_arrow( - "list[orcapod.uuid]", pa.large_list(pa.large_binary()), metadata, _StubConverter() - ) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_extension_types/test_list_logical_type.py -k "factory" -v -``` - -Expected: `FAILED` with `ImportError` or `AttributeError` - -- [ ] **Step 3: Add `ListLogicalTypeFactory` to `list_logical_type_factory.py`** - -Append to `src/orcapod/extension_types/list_logical_type_factory.py`: - -```python -class ListLogicalTypeFactory: - """Stateless factory that reconstructs ``ListLogicalType`` instances from Arrow metadata. - - Registered for categories ``"list"`` and ``"set"`` in the ``LogicalTypeRegistry``. - No ``python_bases`` are registered — write-path dispatch is explicit in - ``UniversalTypeConverter._register_python_class_impl`` and ``_convert_python_to_arrow``. - - Read path only (``reconstruct_from_arrow``). ``create_for_python_type`` raises - ``NotImplementedError`` because explicit dispatch makes it unnecessary. - - Register with:: - - converter.register_logical_type_factory(ListLogicalTypeFactory(), category="list") - converter.register_logical_type_factory(ListLogicalTypeFactory(), category="set") - """ - - def supports_class(self, python_type: type) -> bool: - """Always ``False`` — write-path dispatch is explicit, not via base-class matching. - - Args: - python_type: Ignored. - - Returns: - ``False``. - """ - return False - - def create_for_python_type( - self, - python_type: type, - converter: "TypeConverterProtocol", - ) -> ListLogicalType: - """Not implemented — list/set types are registered directly in the converter. - - Raises: - NotImplementedError: Always. - """ - raise NotImplementedError( - "ListLogicalTypeFactory does not implement create_for_python_type. " - "list[T] and set[T] logical types are created explicitly in " - "UniversalTypeConverter._register_python_class_impl." - ) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: "pa.DataType", - metadata: dict[str, Any], - converter: "TypeConverterProtocol", - ) -> ListLogicalType: - """Reconstruct a ``ListLogicalType`` from Arrow schema metadata (read path). - - Recursively calls ``converter.register_arrow_extension`` for the element type, - ensuring the element logical type is registered before constructing the outer - ``ListLogicalType``. Handles arbitrary nesting depth via recursion. - - Args: - arrow_extension_name: Extension name (e.g. ``"list[orcapod.uuid]"``). - storage_type: Outer storage type (``large_list()``). - metadata: Parsed metadata dict; must contain ``"category"`` and - ``"element_ext_name"``; ``"element_ext_metadata"`` may be ``None``. - converter: Active converter for recursive element registration. - - Returns: - A ``ListLogicalType`` ready for registration. - - Raises: - ValueError: If ``storage_type`` is not a list type, or required metadata - keys are missing. - """ - if not (pa.types.is_large_list(storage_type) or pa.types.is_list(storage_type)): - raise ValueError( - f"ListLogicalTypeFactory.reconstruct_from_arrow: expected a list storage " - f"type for {arrow_extension_name!r}, got {storage_type!r}." - ) - - element_ext_name = metadata.get("element_ext_name") - if not element_ext_name: - raise ValueError( - f"ListLogicalTypeFactory.reconstruct_from_arrow: missing 'element_ext_name' " - f"in metadata for {arrow_extension_name!r}. metadata={metadata!r}." - ) - - element_meta_str = metadata.get("element_ext_metadata") - element_meta_bytes = ( - element_meta_str.encode("utf-8") if element_meta_str else b"" - ) - # Element storage is the value type of the outer list storage. - element_storage_type = storage_type.value_type - - # Recursively register the element logical type (handles nesting). - element_ext_arrow_type = converter.register_arrow_extension( - element_ext_name, element_meta_bytes, element_storage_type - ) - - # Recover element Python type from the now-registered extension type. - element_python_type = converter.arrow_type_to_python_type(element_ext_arrow_type) - - is_set = metadata.get("category") == SET_CATEGORY - logger.debug( - "ListLogicalTypeFactory: reconstructed %r from Arrow (is_set=%s)", - arrow_extension_name, - is_set, - ) - return ListLogicalType(element_python_type, element_ext_arrow_type, is_set=is_set) -``` - -- [ ] **Step 4: Run factory tests to verify they pass** - -```bash -uv run pytest tests/test_extension_types/test_list_logical_type.py -v -``` - -Expected: all `PASSED` - -- [ ] **Step 5: Commit** - -```bash -git add src/orcapod/extension_types/list_logical_type_factory.py tests/test_extension_types/test_list_logical_type.py -git commit -m "feat(extension_types): add ListLogicalTypeFactory for read-path reconstruction" -``` - ---- - -## Task 5: Update `universal_converter.py` — `_register_python_class_impl` - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Test: `tests/test_semantic_types/test_universal_converter.py` (create or modify as appropriate) - -Replace the `ValueError` raises in `_register_python_class_impl` for `list[T]`/`set[T]` when `T` resolves to an extension type. Also add the shared `_make_or_get_list_logical_type` helper. - -- [ ] **Step 1: Write the failing tests** - -Check if `tests/test_semantic_types/test_universal_converter.py` exists; create it if not, or append to existing. Add: - -```python -def test_register_python_class_list_of_uuid_returns_extension_type(): - """register_python_class(list[uuid.UUID]) must return a pa.ExtensionType, not raise.""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - result = converter.register_python_class(list[uuid.UUID]) - - assert isinstance(result, pa.ExtensionType), ( - f"Expected pa.ExtensionType, got {type(result)}: {result!r}" - ) - assert result.extension_name == "list[orcapod.uuid]" - - -def test_register_python_class_set_of_uuid_returns_extension_type(): - """register_python_class(set[uuid.UUID]) must return a pa.ExtensionType.""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - result = converter.register_python_class(set[uuid.UUID]) - - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "set[orcapod.uuid]" - - -def test_register_python_class_list_of_int_unchanged(): - """register_python_class(list[int]) must still return plain large_list(int64).""" - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - result = converter.register_python_class(list[int]) - - assert not isinstance(result, pa.ExtensionType) - assert pa.types.is_large_list(result) - assert result.value_type == pa.int64() - - -def test_register_python_class_list_of_uuid_idempotent(): - """Calling register_python_class(list[uuid.UUID]) twice returns the same ext type.""" - import uuid - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - result1 = converter.register_python_class(list[uuid.UUID]) - result2 = converter.register_python_class(list[uuid.UUID]) - assert result1.extension_name == result2.extension_name -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/ -k "list_of_uuid or set_of_uuid or list_of_int_unchanged" -v -``` - -Expected: `FAILED` with `ValueError: 'list[...]' is not yet supported` - -- [ ] **Step 3: Add `_make_or_get_list_logical_type` helper and update the two branches** - -In `src/orcapod/semantic_types/universal_converter.py`: - -**3a. Add the helper method** (insert after `_find_factory_for_class`, around line 490): - -```python -def _make_or_get_list_logical_type( - self, - element_ext_type: "pa.ExtensionType", - is_set: bool, -) -> "pa.ExtensionType": - """Return (creating and registering if needed) a ``ListLogicalType`` for a container. - - Shared by ``_register_python_class_impl`` and ``_convert_python_to_arrow`` to - ensure idempotent creation — looking up by extension name first avoids - creating two different ``ListLogicalType`` instances for the same annotation. - - Args: - element_ext_type: Arrow extension type of the element. - is_set: ``True`` for ``set[T]``, ``False`` for ``list[T]``. - - Returns: - The ``pa.ExtensionType`` of the created-or-existing ``ListLogicalType``. - """ - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - - prefix = "set" if is_set else "list" - list_ext_name = f"{prefix}[{element_ext_type.extension_name}]" - - # Idempotency: look up by extension name (GenericAlias key not yet in registry). - lt = self._logical_type_registry.get_by_arrow_extension_name(list_ext_name) - if lt is None: - element_python_type = self.arrow_type_to_python_type(element_ext_type) - lt = ListLogicalType(element_python_type, element_ext_type, is_set=is_set) - self._logical_type_registry.register_logical_type(lt) - return lt.get_arrow_extension_type() -``` - -**3b. Replace the `origin is list` block in `_register_python_class_impl`** (around lines 347–361): - -```python -# BEFORE: - if origin is list: - if not args: - raise ValueError( - "Unparameterized 'list' is not supported. Use 'list[T]' with a concrete " - "element type (e.g. list[int], list[str])." - ) - inner = self.register_python_class(args[0]) - if isinstance(inner, pa.ExtensionType): - raise ValueError( - f"'list[{args[0]}]' is not yet supported: the element type maps to Arrow " - f"extension type {inner.extension_name!r}, which cannot be preserved inside " - f"a list value field due to an Arrow limitation (ET2 in DESIGN_ISSUES.md). " - f"Native list-of-logical-type support is tracked in PLT-1732." - ) - return pa.large_list(inner) - -# AFTER: - if origin is list: - if not args: - raise ValueError( - "Unparameterized 'list' is not supported. Use 'list[T]' with a concrete " - "element type (e.g. list[int], list[str])." - ) - inner = self.register_python_class(args[0]) - if isinstance(inner, pa.ExtensionType): - return self._make_or_get_list_logical_type(inner, is_set=False) - return pa.large_list(inner) -``` - -**3c. Replace the `origin is set` block in `_register_python_class_impl`** (around lines 363–378): - -```python -# BEFORE: - if origin is set: - if not args: - raise ValueError( - "Unparameterized 'set' is not supported. Use 'set[T]' with a concrete " - "element type (e.g. set[int], set[str])." - ) - inner = self.register_python_class(args[0]) - if isinstance(inner, pa.ExtensionType): - raise ValueError( - f"'set[{args[0]}]' is not yet supported: the element type maps to Arrow " - f"extension type {inner.extension_name!r}, which cannot be preserved inside " - f"a list value field due to an Arrow limitation (ET2 in DESIGN_ISSUES.md). " - f"Native set-of-logical-type support is tracked in PLT-1732." - ) - return pa.large_list(inner) - -# AFTER: - if origin is set: - if not args: - raise ValueError( - "Unparameterized 'set' is not supported. Use 'set[T]' with a concrete " - "element type (e.g. set[int], set[str])." - ) - inner = self.register_python_class(args[0]) - if isinstance(inner, pa.ExtensionType): - return self._make_or_get_list_logical_type(inner, is_set=True) - return pa.large_list(inner) -``` - -- [ ] **Step 4: Run the new tests** - -```bash -uv run pytest tests/test_semantic_types/ -k "list_of_uuid or set_of_uuid or list_of_int_unchanged" -v -``` - -Expected: all `PASSED` - -- [ ] **Step 5: Run the full test suite to check for regressions** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: no new failures - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/ -git commit -m "feat(converter): register list[T]/set[T] as ListLogicalType when T is an extension type" -``` - ---- - -## Task 6: Update `_convert_python_to_arrow` — remove `isinstance` guard + update list/set branches - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Test: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Write failing tests** - -Append to the test file: - -```python -def test_python_type_to_arrow_type_list_of_uuid_returns_extension_type(): - """python_type_to_arrow_type(list[UUID]) must return the list[orcapod.uuid] ext type.""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - # Pre-register so the type is in the registry before python_type_to_arrow_type is called. - converter.register_python_class(list[uuid.UUID]) - result = converter.python_type_to_arrow_type(list[uuid.UUID]) - - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "list[orcapod.uuid]" - - -def test_python_type_to_arrow_type_list_of_uuid_without_prior_registration(): - """python_type_to_arrow_type(list[UUID]) must work even without prior register_python_class.""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - # Fresh converter — ListLogicalType not yet registered - converter = create_registry().get_context().type_converter - result = converter.python_type_to_arrow_type(list[uuid.UUID]) - - assert isinstance(result, pa.ExtensionType) - assert result.extension_name == "list[orcapod.uuid]" - - -def test_arrow_schema_to_python_schema_round_trip_list_of_uuid(): - """Schema round-trip: list[UUID] → Arrow ext → python schema → Arrow ext (same type).""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - python_schema = {"ids": list[uuid.UUID]} - arrow_schema = converter.python_schema_to_arrow_schema(python_schema) - - # Arrow schema has list[orcapod.uuid] extension type - assert arrow_schema.field("ids").type.extension_name == "list[orcapod.uuid]" - - # Recover Python schema - recovered = converter.arrow_schema_to_python_schema(arrow_schema) - assert recovered["ids"] == list[uuid.UUID] - - # Re-derive Arrow schema — must be identical - arrow_schema2 = converter.python_schema_to_arrow_schema(recovered) - assert arrow_schema2.field("ids").type.extension_name == "list[orcapod.uuid]" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/test_semantic_types/ -k "python_type_to_arrow_type_list or arrow_schema_to_python_schema_round_trip_list" -v -``` - -Expected: `FAILED` — `python_type_to_arrow_type(list[UUID])` returns plain `large_list` instead of the ext type, because the early registry check skips `GenericAlias`. - -- [ ] **Step 3: Remove `isinstance(python_type, type)` guard in `_convert_python_to_arrow`** - -Find the early registry check in `_convert_python_to_arrow` (around line 1079): - -```python -# BEFORE: - if self._logical_type_registry is not None and isinstance(python_type, type): - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - return lt.get_arrow_extension_type() - -# AFTER: - if self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - return lt.get_arrow_extension_type() -``` - -Also update the `origin is list` branch in `_convert_python_to_arrow` (around line 1100) to create `ListLogicalType` when element is an ext type: - -```python -# BEFORE: - if origin is list: - if len(args) != 1: - raise ValueError( - f"list type must have exactly one type argument, got: {args}" - ) - element_type = self.python_type_to_arrow_type(args[0]) - return pa.large_list(element_type) - -# AFTER: - if origin is list: - if len(args) != 1: - raise ValueError( - f"list type must have exactly one type argument, got: {args}" - ) - element_type = self.python_type_to_arrow_type(args[0]) - if isinstance(element_type, pa.ExtensionType) and self._logical_type_registry is not None: - return self._make_or_get_list_logical_type(element_type, is_set=False) - return pa.large_list(element_type) -``` - -And the `origin is set` branch in `_convert_python_to_arrow` (around line 1169): - -```python -# BEFORE: - elif origin is set: - if len(args) != 1: - raise ValueError( - f"set type must have exactly one type argument, got: {args}" - ) - element_type = self.python_type_to_arrow_type(args[0]) - return pa.large_list(element_type) - -# AFTER: - elif origin is set: - if len(args) != 1: - raise ValueError( - f"set type must have exactly one type argument, got: {args}" - ) - element_type = self.python_type_to_arrow_type(args[0]) - if isinstance(element_type, pa.ExtensionType) and self._logical_type_registry is not None: - return self._make_or_get_list_logical_type(element_type, is_set=True) - return pa.large_list(element_type) -``` - -- [ ] **Step 4: Run the new tests** - -```bash -uv run pytest tests/test_semantic_types/ -k "python_type_to_arrow_type_list or arrow_schema_to_python_schema_round_trip_list" -v -``` - -Expected: all `PASSED` - -- [ ] **Step 5: Run full suite** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: no new failures - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/ -git commit -m "fix(converter): remove isinstance guard in _convert_python_to_arrow to handle GenericAlias registry lookup" -``` - ---- - -## Task 7: Update `_create_python_to_arrow_converter` — remove `isinstance` guard - -**Files:** -- Modify: `src/orcapod/semantic_types/universal_converter.py` -- Test: `tests/test_semantic_types/test_universal_converter.py` - -- [ ] **Step 1: Write failing test** - -```python -def test_value_converter_list_of_uuid_produces_bytes_list(): - """get_python_to_arrow_converter(list[UUID]) converts [uuid, uuid] → [bytes, bytes].""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - converter.register_python_class(list[uuid.UUID]) - - conv_fn = converter.get_python_to_arrow_converter(list[uuid.UUID]) - u1 = uuid.UUID("12345678-1234-5678-1234-567812345678") - u2 = uuid.UUID("87654321-4321-8765-4321-876543218765") - result = conv_fn([u1, u2]) - - assert result == [u1.bytes, u2.bytes] - - -def test_value_converter_set_of_uuid_produces_bytes_list(): - """get_python_to_arrow_converter(set[UUID]) converts {uuid} → [bytes].""" - import uuid - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - converter.register_python_class(set[uuid.UUID]) - - conv_fn = converter.get_python_to_arrow_converter(set[uuid.UUID]) - u1 = uuid.UUID("12345678-1234-5678-1234-567812345678") - result = conv_fn({u1}) - - assert result == [u1.bytes] -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/test_semantic_types/ -k "value_converter_list_of_uuid or value_converter_set_of_uuid" -v -``` - -Expected: `FAILED` — converter returns the list/set unchanged (passthrough) instead of converting UUIDs to bytes. - -- [ ] **Step 3: Remove `isinstance(python_type, type)` guard in `_create_python_to_arrow_converter`** - -Find the early registry check in `_create_python_to_arrow_converter` (around line 1391): - -```python -# BEFORE: - if self._logical_type_registry is not None and isinstance(python_type, type): - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - _lt = lt - _self = self - return lambda value: _lt.python_to_storage(value, _self) - -# AFTER: - if self._logical_type_registry is not None: - lt = self._logical_type_registry.get_by_python_type(python_type) - if lt is not None: - _lt = lt - _self = self - return lambda value: _lt.python_to_storage(value, _self) -``` - -- [ ] **Step 4: Run the new tests** - -```bash -uv run pytest tests/test_semantic_types/ -k "value_converter_list_of_uuid or value_converter_set_of_uuid" -v -``` - -Expected: all `PASSED` - -- [ ] **Step 5: Run full suite** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: no new failures - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/semantic_types/universal_converter.py tests/test_semantic_types/ -git commit -m "fix(converter): remove isinstance guard in _create_python_to_arrow_converter for GenericAlias support" -``` - ---- - -## Task 8: Wire up exports and `v0.1.json` - -**Files:** -- Modify: `src/orcapod/extension_types/__init__.py` -- Modify: `src/orcapod/contexts/data/v0.1.json` - -- [ ] **Step 1: Add exports to `__init__.py`** - -In `src/orcapod/extension_types/__init__.py`, add the import after the `LogicalPandasSeries` import line: - -```python -from .list_logical_type_factory import LIST_CATEGORY, SET_CATEGORY, ListLogicalType, ListLogicalTypeFactory -``` - -And add to `__all__`: - -```python - # ITL-173 - "LIST_CATEGORY", - "SET_CATEGORY", - "ListLogicalType", - "ListLogicalTypeFactory", -``` - -- [ ] **Step 2: Register the factory in `v0.1.json`** - -In `src/orcapod/contexts/data/v0.1.json`, find the `"factories"` array and add two entries after the existing `pydantic` factory entry: - -```json - { - "factory": { - "_class": "orcapod.extension_types.list_logical_type_factory.ListLogicalTypeFactory", - "_config": {} - }, - "category": "list" - }, - { - "factory": { - "_class": "orcapod.extension_types.list_logical_type_factory.ListLogicalTypeFactory", - "_config": {} - }, - "category": "set" - } -``` - -- [ ] **Step 3: Verify the default context loads cleanly** - -```bash -uv run python -c " -from orcapod.contexts import get_default_context -ctx = get_default_context() -print('context_key:', ctx.context_key) -print('list factory registered:', ctx.type_converter._logical_type_registry._category_factories.get('list')) -print('set factory registered:', ctx.type_converter._logical_type_registry._category_factories.get('set')) -" -``` - -Expected output: -``` -context_key: std:v0.1:default -list factory registered: -set factory registered: -``` - -- [ ] **Step 4: Verify public import works** - -```bash -uv run python -c " -from orcapod.extension_types import ListLogicalType, ListLogicalTypeFactory, LIST_CATEGORY, SET_CATEGORY -print('LIST_CATEGORY:', LIST_CATEGORY) -print('SET_CATEGORY:', SET_CATEGORY) -print('OK') -" -``` - -Expected: `LIST_CATEGORY: list`, `SET_CATEGORY: set`, `OK` - -- [ ] **Step 5: Run full suite** - -```bash -uv run pytest tests/ -x -q 2>&1 | tail -20 -``` - -Expected: no failures - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/extension_types/__init__.py src/orcapod/contexts/data/v0.1.json -git commit -m "feat(extension_types): export ListLogicalType/Factory and register in v0.1 context" -``` - ---- - -## Task 9: Integration round-trip tests - -**Files:** -- Modify: `tests/test_extension_types/test_roundtrips.py` - -Add module-level dataclasses and 9 new test functions covering all spec test cases. - -- [ ] **Step 1: Add module-level dataclasses** (after existing `_Outer` definition) - -```python -@dataclasses.dataclass -class _TaggedPoint: - """Dataclass with a list[uuid.UUID] field — tests ET2 fix in DataclassLogicalTypeFactory.""" - name: str - ids: list[uuid_module.UUID] - - -@dataclasses.dataclass -class _SimplePoint: - """Dataclass with only scalar fields — used as element of list[_SimplePoint].""" - label: str - value: int -``` - -- [ ] **Step 2: Write all 9 failing tests** - -Append to `tests/test_extension_types/test_roundtrips.py`: - -```python -# ── list[T] / set[T] round-trip tests (ITL-173) ────────────────────────────── - - -def test_list_of_uuid_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """list[uuid.UUID] round-trips with extension name list[orcapod.uuid].""" - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - u2 = uuid_module.UUID("87654321-4321-8765-4321-876543218765") - - result, read_converter = _write_and_read( - {"ids": list[uuid_module.UUID]}, - [{"ids": [u1, u2]}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("ids") - assert hasattr(field.type, "extension_name"), ( - f"Expected extension type on 'ids', got {field.type!r}" - ) - assert field.type.extension_name == "list[orcapod.uuid]" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert rows[0]["ids"] == [u1, u2] - assert all(isinstance(v, uuid_module.UUID) for v in rows[0]["ids"]) - - -def test_set_of_uuid_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """set[uuid.UUID] round-trips with extension name set[orcapod.uuid]; read back as set.""" - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - u2 = uuid_module.UUID("87654321-4321-8765-4321-876543218765") - - result, read_converter = _write_and_read( - {"ids": set[uuid_module.UUID]}, - [{"ids": {u1, u2}}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("ids") - assert hasattr(field.type, "extension_name") - assert field.type.extension_name == "set[orcapod.uuid]" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert isinstance(rows[0]["ids"], set) - assert rows[0]["ids"] == {u1, u2} - - -def test_list_of_dataclass_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """list[_SimplePoint] round-trips — ListLogicalType wrapping DataclassLogicalType.""" - p1 = _SimplePoint(label="alpha", value=1) - p2 = _SimplePoint(label="beta", value=2) - - result, read_converter = _write_and_read( - {"points": list[_SimplePoint]}, - [{"points": [p1, p2]}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("points") - assert hasattr(field.type, "extension_name") - assert field.type.extension_name.startswith("list[") - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - reconstructed = rows[0]["points"] - assert len(reconstructed) == 2 - assert reconstructed[0] == p1 - assert reconstructed[1] == p2 - - -def test_list_of_list_of_uuid_round_trip(storage_backend: _StorageBackend, tmp_path: Path) -> None: - """list[list[uuid.UUID]] round-trips — two-level nesting.""" - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - u2 = uuid_module.UUID("87654321-4321-8765-4321-876543218765") - - result, read_converter = _write_and_read( - {"groups": list[list[uuid_module.UUID]]}, - [{"groups": [[u1, u2], [u2]]}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("groups") - assert hasattr(field.type, "extension_name") - assert field.type.extension_name == "list[list[orcapod.uuid]]" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - assert rows[0]["groups"] == [[u1, u2], [u2]] - - -def test_dataclass_with_list_uuid_field_round_trip( - storage_backend: _StorageBackend, tmp_path: Path -) -> None: - """Dataclass with list[uuid.UUID] field round-trips (previously broke DataclassLogicalTypeFactory). - - Before this fix, registering _TaggedPoint raised ValueError because - register_python_class(list[uuid.UUID]) was not yet supported. - """ - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - u2 = uuid_module.UUID("87654321-4321-8765-4321-876543218765") - obj = _TaggedPoint(name="test", ids=[u1, u2]) - - result, read_converter = _write_and_read( - {"data": _TaggedPoint}, - [{"data": obj}], - storage_backend, - tmp_path, - ) - - field = result.schema.field("data") - assert hasattr(field.type, "extension_name") - - rows = read_converter.arrow_table_to_python_dicts(result) - assert len(rows) == 1 - reconstructed = rows[0]["data"] - assert isinstance(reconstructed, _TaggedPoint) - assert reconstructed.name == "test" - assert reconstructed.ids == [u1, u2] - assert all(isinstance(v, uuid_module.UUID) for v in reconstructed.ids) - - -def test_list_of_int_produces_no_extension_type(tmp_path: Path) -> None: - """list[int] must still produce plain large_list(int64) — no ListLogicalType wrapping.""" - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - result = converter.register_python_class(list[int]) - - assert not isinstance(result, pa.ExtensionType), ( - f"list[int] must not be wrapped as an extension type, got {result!r}" - ) - assert pa.types.is_large_list(result) - assert result.value_type == pa.int64() - - -def test_schema_round_trip_list_of_uuid(tmp_path: Path) -> None: - """arrow_schema_to_python_schema then python_schema_to_arrow_schema is identity for list[UUID].""" - import uuid - import pyarrow as pa - from orcapod.contexts import create_registry - - converter = create_registry().get_context().type_converter - - python_schema = { - "ids": list[uuid.UUID], - "groups": list[list[uuid.UUID]], - "tag_set": set[uuid.UUID], - } - arrow_schema = converter.python_schema_to_arrow_schema(python_schema) - - recovered_python = converter.arrow_schema_to_python_schema(arrow_schema) - assert recovered_python["ids"] == list[uuid.UUID] - assert recovered_python["groups"] == list[list[uuid.UUID]] - assert recovered_python["tag_set"] == set[uuid.UUID] - - arrow_schema2 = converter.python_schema_to_arrow_schema(recovered_python) - assert arrow_schema2.field("ids").type.extension_name == "list[orcapod.uuid]" - assert arrow_schema2.field("groups").type.extension_name == "list[list[orcapod.uuid]]" - assert arrow_schema2.field("tag_set").type.extension_name == "set[orcapod.uuid]" - - -def test_python_type_property_list_and_set(tmp_path: Path) -> None: - """ListLogicalType.python_type returns the exact generic alias.""" - import uuid - from orcapod.extension_types.builtin_logical_types import LogicalUUID - from orcapod.extension_types.list_logical_type_factory import ListLogicalType - - uuid_ext = LogicalUUID().get_arrow_extension_type() - - lt_list = ListLogicalType(uuid.UUID, uuid_ext, is_set=False) - assert lt_list.python_type == list[uuid.UUID] - - lt_set = ListLogicalType(uuid.UUID, uuid_ext, is_set=True) - assert lt_set.python_type == set[uuid.UUID] - - -def test_fresh_converter_reads_list_of_uuid( - storage_backend: _StorageBackend, tmp_path: Path -) -> None: - """A fresh converter (no prior registration) can read list[UUID] via load_extension_types.""" - u1 = uuid_module.UUID("12345678-1234-5678-1234-567812345678") - - # Write with converter A - write_converter = _fresh_converter() - write_converter.register_python_class(list[uuid_module.UUID]) - arrow_schema = write_converter.python_schema_to_arrow_schema({"ids": list[uuid_module.UUID]}) - table = write_converter.python_dicts_to_arrow_table([{"ids": [u1]}], arrow_schema=arrow_schema) - storage_backend.write(table, tmp_path) - - # Read with converter B — no prior registration; load_extension_types triggers factory - read_converter = _fresh_converter() - result = storage_backend.read(tmp_path, read_converter) - - field = result.schema.field("ids") - assert hasattr(field.type, "extension_name") - assert field.type.extension_name == "list[orcapod.uuid]" - - rows = read_converter.arrow_table_to_python_dicts(result) - assert rows[0]["ids"] == [u1] - assert isinstance(rows[0]["ids"][0], uuid_module.UUID) -``` - -- [ ] **Step 3: Run tests to verify they fail (before implementation is complete)** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py -k "list_of_uuid or set_of_uuid or list_of_dataclass or list_of_list or dataclass_with_list or list_of_int or schema_round_trip_list or python_type_property or fresh_converter_reads" -v 2>&1 | tail -30 -``` - -Expected: most `FAILED` (we haven't wired up `v0.1.json` yet — wait, we did that in Task 8. So actually these should mostly pass now. If Task 8 was done first, these may all pass immediately.) - -- [ ] **Step 4: Run all round-trip tests** - -```bash -uv run pytest tests/test_extension_types/test_roundtrips.py -v -``` - -Expected: all `PASSED` - -- [ ] **Step 5: Run the full test suite** - -```bash -uv run pytest tests/ -q 2>&1 | tail -20 -``` - -Expected: all `PASSED` - -- [ ] **Step 6: Commit** - -```bash -git add tests/test_extension_types/test_roundtrips.py -git commit -m "test(roundtrips): add list[T], set[T], list[list[T]], and dataclass-with-list-field round-trip tests" -``` - ---- - -## Task 10: Update `DESIGN_ISSUES.md` and commit spec - -**Files:** -- Modify: `DESIGN_ISSUES.md` -- Modify: `superpowers/specs/2026-08-11-itl-173-list-set-logical-types-design.md` (status update) - -- [ ] **Step 1: Update ET2 entry in `DESIGN_ISSUES.md`** - -Find the ET2 entry (search for `ET2` or `list[T]`). Update status to `resolved` and add a Fix note: - -``` -**Status:** resolved - -**Fix:** Introduced `ListLogicalType` and `ListLogicalTypeFactory` in -`src/orcapod/extension_types/list_logical_type_factory.py`. The entire -`list[T]`/`set[T]` is wrapped as a top-level Arrow extension type with storage -`large_list()`, embedding element reconstruction metadata in the field-level -metadata JSON. Wired into `_register_python_class_impl` and `_convert_python_to_arrow` -via `_make_or_get_list_logical_type`. Registered in `v0.1.json` under categories -`"list"` and `"set"`. Closes ITL-173. -``` - -- [ ] **Step 2: Run full test suite one final time** - -```bash -uv run pytest tests/ -q 2>&1 | tail -10 -``` - -Expected: all `PASSED`, zero failures - -- [ ] **Step 3: Commit** - -```bash -git add DESIGN_ISSUES.md superpowers/specs/2026-08-11-itl-173-list-set-logical-types-design.md -git commit -m "docs: mark ET2 resolved in DESIGN_ISSUES.md; update spec status" -``` - ---- - -## Self-Review - -**Spec coverage check:** - -| Spec requirement | Task(s) | -|---|---| -| `list[T]` `register_python_class` creates `ListLogicalType` | Task 5 | -| `set[T]` same | Task 5 | -| `_convert_python_to_arrow` GenericAlias registry lookup | Task 6 | -| `_convert_python_to_arrow` list/set branches call `_make_or_get_list_logical_type` | Task 6 | -| `_create_python_to_arrow_converter` GenericAlias registry lookup | Task 7 | -| `ListLogicalType` construction, `python_type` generic alias | Task 3 | -| `ListLogicalType` ET1 safe storage (`large_list()`) | Task 3 | -| `ListLogicalTypeFactory.reconstruct_from_arrow` recursive element registration | Task 4 | -| `registry.py` `__qualname__` guard | Task 1 | -| `TypeConverterProtocol.arrow_type_to_python_type` | Task 2 | -| Exports and `v0.1.json` registration | Task 8 | -| Test: `list[UUID]` round-trip | Task 9 | -| Test: `set[UUID]` round-trip | Task 9 | -| Test: `list[MyDataclass]` round-trip | Task 9 | -| Test: `list[list[UUID]]` round-trip | Task 9 | -| Test: dataclass with `list[UUID]` field | Task 9 | -| Test: `list[int]` unchanged | Task 9 | -| Test: schema round-trip | Task 9 | -| Test: `python_type` property | Task 9 | -| Test: fresh converter read | Task 9 | -| `DESIGN_ISSUES.md` ET2 resolved | Task 10 | - -All spec requirements covered. No placeholders found. Type names consistent throughout (`ListLogicalType`, `ListLogicalTypeFactory`, `LIST_CATEGORY`, `SET_CATEGORY`, `_make_or_get_list_logical_type`). diff --git a/superpowers/plans/2026-08-20-github-actions-node24-upgrade.md b/superpowers/plans/2026-08-20-github-actions-node24-upgrade.md deleted file mode 100644 index 3f0c89d2c..000000000 --- a/superpowers/plans/2026-08-20-github-actions-node24-upgrade.md +++ /dev/null @@ -1,545 +0,0 @@ -# GitHub Actions Node 24 Upgrade — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Upgrade all five third-party GitHub Actions from Node 20 majors to Node 24 majors, eliminating the deprecation warning and the risk of CI breaking when the GitHub runner shim is withdrawn. - -**Architecture:** Pure workflow YAML edits — replace the pinned SHA and `# vX.Y.Z` comment for each action across all affected workflow files. No Python code changes. Five commits ordered by risk (smallest version jump first), all in one PR targeting `main`. - -**Tech Stack:** GitHub Actions YAML, `sed`, `grep`, `python -c "import yaml"` for syntax validation. - ---- - -## Files modified - -| File | Actions touched | -|---|---| -| `.github/workflows/run-tests.yml` | all five | -| `.github/workflows/run-objective-tests.yml` | setup-python, codecov-action, checkout, setup-uv | -| `.github/workflows/run-postgres-tests.yml` | setup-python, codecov-action, checkout, setup-uv | -| `.github/workflows/tests.yml` | checkout, setup-uv | -| `.github/workflows/_license-check.yml` | checkout, setup-uv | -| `.github/workflows/release.yml` | checkout, setup-uv | -| `.github/workflows/release-sync.yml` | checkout | - ---- - -## Task 0: Create the feature branch - -**Files:** none - -- [ ] **Step 1: Check out the branch** - -```bash -git checkout -b eywalker/itl-621-upgrade-all-pinned-github-actions-to-current-major-versions -``` - -- [ ] **Step 2: Verify you are on the correct branch** - -```bash -git branch --show-current -``` - -Expected output: -``` -eywalker/itl-621-upgrade-all-pinned-github-actions-to-current-major-versions -``` - ---- - -## Task 1: Upgrade `actions/dependency-review-action` v4.9.0 → v5.0.0 - -**Files:** -- Modify: `.github/workflows/run-tests.yml` - -**Breaking changes confirmed no-op:** Node 20→24 runtime only. The `deny-licenses` list is unchanged and carries over. - -- [ ] **Step 1: Apply the replacement** - -```bash -sed -i 's|actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0|actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0|g' \ - .github/workflows/run-tests.yml -``` - -- [ ] **Step 2: Verify old SHA is gone** - -```bash -grep -r "2031cfc080254a8a887f58cffee85186f0e49e48" .github/workflows/ -``` - -Expected: no output (no matches). - -- [ ] **Step 3: Verify new SHA is present** - -```bash -grep -r "a1d282b36b6f3519aa1f3fc636f609c47dddb294" .github/workflows/ -``` - -Expected: -``` -.github/workflows/run-tests.yml: uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 -``` - -- [ ] **Step 4: Validate YAML syntax** - -```bash -python3 -c "import yaml, sys; yaml.safe_load(open('.github/workflows/run-tests.yml'))" && echo "OK" -``` - -Expected: `OK` - -- [ ] **Step 5: Commit** - -```bash -git add .github/workflows/run-tests.yml -git commit -m "ci: upgrade dependency-review-action to v5.0.0 (ITL-621) - -Node 20→24 runtime upgrade only. deny-licenses list unchanged — no-op." -``` - ---- - -## Task 2: Upgrade `actions/setup-python` v5.6.0 → v7.0.0 - -**Files:** -- Modify: `.github/workflows/run-tests.yml` (2 occurrences: test job, spiral-integration job) -- Modify: `.github/workflows/run-objective-tests.yml` (1 occurrence) -- Modify: `.github/workflows/run-postgres-tests.yml` (1 occurrence) - -**Breaking changes confirmed no-op:** -- v6: Node 20→24. -- v7: `pip-install` input removed — we don't use it; uv handles all package installs. - -- [ ] **Step 1: Apply the replacement across all three files** - -```bash -sed -i 's|actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0|actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0|g' \ - .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml -``` - -- [ ] **Step 2: Verify old SHA is gone** - -```bash -grep -r "a26af69be951a213d495a4c3e4e4022e16d87065" .github/workflows/ -``` - -Expected: no output. - -- [ ] **Step 3: Verify new SHA appears in all three files** - -```bash -grep -r "5fda3b95a4ea91299a34e894583c3862153e4b97" .github/workflows/ -``` - -Expected (4 lines — 2 in run-tests.yml, 1 each in the other two): -``` -.github/workflows/run-objective-tests.yml: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 -.github/workflows/run-postgres-tests.yml: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 -.github/workflows/run-tests.yml: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 -.github/workflows/run-tests.yml: uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 -``` - -- [ ] **Step 4: Validate YAML syntax for all three files** - -```bash -for f in .github/workflows/run-tests.yml .github/workflows/run-objective-tests.yml .github/workflows/run-postgres-tests.yml; do - python3 -c "import yaml; yaml.safe_load(open('$f'))" && echo "OK: $f" -done -``` - -Expected: three `OK` lines. - -- [ ] **Step 5: Commit** - -```bash -git add .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml -git commit -m "ci: upgrade setup-python to v7.0.0 (ITL-621) - -v6: Node 20→24. v7: pip-install input removed — we don't use it (uv handles installs)." -``` - ---- - -## Task 3: Upgrade `codecov/codecov-action` v5.5.5 → v7.0.0 - -**Files:** -- Modify: `.github/workflows/run-tests.yml` (1 occurrence: test job) -- Modify: `.github/workflows/run-objective-tests.yml` (1 occurrence) -- Modify: `.github/workflows/run-postgres-tests.yml` (1 occurrence) - -**Breaking changes confirmed no-op:** -- v6: Node 24 support. -- v7: GPG signing key rotated from `codecovsecurity` to `codecovsecops` — we don't verify Codecov signatures. - -- [ ] **Step 1: Apply the replacement across all three files** - -```bash -sed -i 's|codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5|codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0|g' \ - .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml -``` - -- [ ] **Step 2: Verify old SHA is gone** - -```bash -grep -r "0fb7174895f61a3b6b78fc075e0cd60383518dac" .github/workflows/ -``` - -Expected: no output. - -- [ ] **Step 3: Verify new SHA appears in all three files** - -```bash -grep -r "fb8b3582c8e4def4969c97caa2f19720cb33a72f" .github/workflows/ -``` - -Expected (3 lines, one per file): -``` -.github/workflows/run-objective-tests.yml: uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 -.github/workflows/run-postgres-tests.yml: uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 -.github/workflows/run-tests.yml: uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 -``` - -- [ ] **Step 4: Validate YAML syntax for all three files** - -```bash -for f in .github/workflows/run-tests.yml .github/workflows/run-objective-tests.yml .github/workflows/run-postgres-tests.yml; do - python3 -c "import yaml; yaml.safe_load(open('$f'))" && echo "OK: $f" -done -``` - -Expected: three `OK` lines. - -- [ ] **Step 5: Commit** - -```bash -git add .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml -git commit -m "ci: upgrade codecov-action to v7.0.0 (ITL-621) - -v6: Node 24 support. v7: GPG key rotation only — we don't verify Codecov signatures." -``` - ---- - -## Task 4: Upgrade `actions/checkout` v4.4.0 → v7.0.0 - -**Files:** -- Modify: `.github/workflows/run-tests.yml` (3 occurrences: test, spiral-integration, dependency-review jobs) -- Modify: `.github/workflows/run-objective-tests.yml` (1 occurrence) -- Modify: `.github/workflows/run-postgres-tests.yml` (1 occurrence) -- Modify: `.github/workflows/tests.yml` (1 occurrence) -- Modify: `.github/workflows/_license-check.yml` (1 occurrence) -- Modify: `.github/workflows/release.yml` (4 occurrences: test, build, linear-sync, linear-complete jobs) -- Modify: `.github/workflows/release-sync.yml` (1 occurrence: sync job) - -**Breaking changes confirmed no-op:** -- v5: Node 20→24. -- v6: Credentials written to a separate file instead of `.git/config`. `release.yml` uses `git push` and `git tag` which both work correctly with the new credential location — nothing reads the extraheader out of `.git/config` directly. -- v7: Blocks checking out the fork PR head for `pull_request_target` and `workflow_run` events. We only use `pull_request`, `push`, and `workflow_dispatch` — no `pull_request_target` anywhere in our workflows. - -- [ ] **Step 1: Apply the replacement across all seven files** - -```bash -sed -i 's|actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0|actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0|g' \ - .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml \ - .github/workflows/tests.yml \ - .github/workflows/_license-check.yml \ - .github/workflows/release.yml \ - .github/workflows/release-sync.yml -``` - -- [ ] **Step 2: Verify old SHA is gone** - -```bash -grep -r "11d5960a326750d5838078e36cf38b85af677262" .github/workflows/ -``` - -Expected: no output. - -- [ ] **Step 3: Verify new SHA count — expect 12 occurrences total** - -```bash -grep -rc "9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" .github/workflows/ -``` - -Expected (files with non-zero counts only): -``` -.github/workflows/_license-check.yml:1 -.github/workflows/release-sync.yml:1 -.github/workflows/release.yml:4 -.github/workflows/run-objective-tests.yml:1 -.github/workflows/run-postgres-tests.yml:1 -.github/workflows/run-tests.yml:3 -.github/workflows/tests.yml:1 -``` - -- [ ] **Step 4: Confirm no `pull_request_target` or `workflow_run` triggers in any workflow** - -```bash -grep -r "pull_request_target\|workflow_run" .github/workflows/ -``` - -Expected: no output (confirms v7 fork-head block is a no-op for us). - -- [ ] **Step 5: Validate YAML syntax for all seven files** - -```bash -for f in .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml \ - .github/workflows/tests.yml \ - .github/workflows/_license-check.yml \ - .github/workflows/release.yml \ - .github/workflows/release-sync.yml; do - python3 -c "import yaml; yaml.safe_load(open('$f'))" && echo "OK: $f" -done -``` - -Expected: seven `OK` lines. - -- [ ] **Step 6: Commit** - -```bash -git add .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml \ - .github/workflows/tests.yml \ - .github/workflows/_license-check.yml \ - .github/workflows/release.yml \ - .github/workflows/release-sync.yml -git commit -m "ci: upgrade actions/checkout to v7.0.0 (ITL-621) - -v5: Node 20→24. v6: creds in separate file — git push/tag in release.yml unaffected. -v7: fork-head block for pull_request_target/workflow_run — we don't use either event." -``` - ---- - -## Task 5: Upgrade `astral-sh/setup-uv` v5.4.2 → v10.0.1 - -**Files:** -- Modify: `.github/workflows/run-tests.yml` (2 occurrences: test, spiral-integration jobs) -- Modify: `.github/workflows/run-objective-tests.yml` (1 occurrence) -- Modify: `.github/workflows/run-postgres-tests.yml` (1 occurrence) -- Modify: `.github/workflows/tests.yml` (1 occurrence) -- Modify: `.github/workflows/_license-check.yml` (1 occurrence) -- Modify: `.github/workflows/release.yml` (4 occurrences: test, build, publish-testpypi, publish-pypi jobs) - -**Breaking changes confirmed no-op:** -- v6: `activate-environment`, `working-directory`, `cache-dependency-glob` defaults changed — we don't pass any of these inputs. -- v7: Node 20→24; `server-url` input removed — we don't use it. -- v8: Immutable releases; major/minor tags gone — we SHA-pin anyway, so no impact. -- v9: `prune-cache` default flips `true` → `false`. **Decision: accept the new default.** Cache entries will grow slightly; monitor after merge (tracked in follow-up issue). -- v10: `enable-cache: auto` disables cache for `pull_request_target`/`workflow_run`. We don't use either event. `tests.yml` uses `enable-cache: true` explicitly — this is unaffected. - -- [ ] **Step 1: Apply the replacement across all six files** - -```bash -sed -i 's|astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2|astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1|g' \ - .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml \ - .github/workflows/tests.yml \ - .github/workflows/_license-check.yml \ - .github/workflows/release.yml -``` - -- [ ] **Step 2: Verify old SHA is gone** - -```bash -grep -r "d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86" .github/workflows/ -``` - -Expected: no output. - -- [ ] **Step 3: Verify new SHA count — expect 10 occurrences total** - -```bash -grep -rc "20cfd1bf945f4377ade1205e4dbc17946fc9a30d" .github/workflows/ -``` - -Expected (files with non-zero counts only): -``` -.github/workflows/_license-check.yml:1 -.github/workflows/release.yml:4 -.github/workflows/run-objective-tests.yml:1 -.github/workflows/run-postgres-tests.yml:1 -.github/workflows/run-tests.yml:2 -.github/workflows/tests.yml:1 -``` - -- [ ] **Step 4: Confirm `enable-cache: true` is still explicit in tests.yml** - -```bash -grep "enable-cache" .github/workflows/tests.yml -``` - -Expected: -``` - enable-cache: true -``` - -- [ ] **Step 5: Validate YAML syntax for all six files** - -```bash -for f in .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml \ - .github/workflows/tests.yml \ - .github/workflows/_license-check.yml \ - .github/workflows/release.yml; do - python3 -c "import yaml; yaml.safe_load(open('$f'))" && echo "OK: $f" -done -``` - -Expected: six `OK` lines. - -- [ ] **Step 6: Commit** - -```bash -git add .github/workflows/run-tests.yml \ - .github/workflows/run-objective-tests.yml \ - .github/workflows/run-postgres-tests.yml \ - .github/workflows/tests.yml \ - .github/workflows/_license-check.yml \ - .github/workflows/release.yml -git commit -m "ci: upgrade setup-uv to v10.0.1 (ITL-621) - -v7: Node 20→24. v8: immutable releases (SHA-pin unaffected). v9: prune-cache default -flips to false — accepted, monitoring via follow-up issue. v10: enable-cache: auto -change for pull_request_target/workflow_run — we don't use either event." -``` - ---- - -## Task 6: Final verification, PR, and follow-up issue - -**Files:** none - -- [ ] **Step 1: Confirm no Node-20 action SHAs remain anywhere in workflow files** - -```bash -# Old SHAs that must all be absent: -for sha in \ - 11d5960a326750d5838078e36cf38b85af677262 \ - d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 \ - a26af69be951a213d495a4c3e4e4022e16d87065 \ - 0fb7174895f61a3b6b78fc075e0cd60383518dac \ - 2031cfc080254a8a887f58cffee85186f0e49e48; do - result=$(grep -r "$sha" .github/workflows/) - if [ -n "$result" ]; then - echo "STILL PRESENT: $sha" - echo "$result" - else - echo "GONE: $sha" - fi -done -``` - -Expected: five `GONE:` lines, no `STILL PRESENT:` lines. - -- [ ] **Step 2: Confirm all five new SHAs are present** - -```bash -for sha in \ - 9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 \ - 20cfd1bf945f4377ade1205e4dbc17946fc9a30d \ - 5fda3b95a4ea91299a34e894583c3862153e4b97 \ - fb8b3582c8e4def4969c97caa2f19720cb33a72f \ - a1d282b36b6f3519aa1f3fc636f609c47dddb294; do - count=$(grep -rc "$sha" .github/workflows/ | awk -F: '{sum += $2} END {print sum}') - echo "SHA $sha: $count occurrences" -done -``` - -Expected: -``` -SHA 9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0: 12 occurrences (checkout, 7 files) -SHA 20cfd1bf945f4377ade1205e4dbc17946fc9a30d: 10 occurrences (setup-uv, 6 files) -SHA 5fda3b95a4ea91299a34e894583c3862153e4b97: 4 occurrences (setup-python, 3 files) -SHA fb8b3582c8e4def4969c97caa2f19720cb33a72f: 3 occurrences (codecov-action, 3 files) -SHA a1d282b36b6f3519aa1f3fc636f609c47dddb294: 1 occurrence (dependency-review-action, 1 file) -``` - -- [ ] **Step 3: Push the branch** - -```bash -git push -u origin eywalker/itl-621-upgrade-all-pinned-github-actions-to-current-major-versions -``` - -- [ ] **Step 4: Create the follow-up Linear issue** - -Use the Linear MCP tool: - -``` -mcp__claude_ai_Linear__save_issue( - title: "Monitor Actions cache size after setup-uv prune-cache default flip", - team: "Tools", - description: "## Overview\n\nITL-621 upgraded setup-uv from v5 to v10. v9.0.0 flipped the `prune-cache`\ndefault from `true` to `false`. We accepted the new default rather than\nback-filling `prune-cache: true` across all six setup-uv call sites.\n\nMonitor the GitHub Actions cache after this lands to determine if the\nlarger cache footprint is acceptable.\n\n## Goals & Success Criteria\n\n* Check Actions cache storage in the repository after a week of CI runs.\n* If cache entries exceed a reasonable threshold, add `prune-cache: true`\n explicitly to all setup-uv call sites in a follow-up commit.\n* Close this issue once the cache behaviour is confirmed acceptable or\n the explicit pin has been added.\n\n## Dependencies & Risks\n\n* Depends on ITL-621 merging first." -) -``` - -- [ ] **Step 5: Open the PR** - -```bash -gh pr create \ - --title "ci: upgrade all pinned GitHub Actions to Node 24 majors (ITL-621)" \ - --base main \ - --body "$(cat <<'EOF' -## Summary - -Upgrades all five third-party GitHub Actions from their Node 20 majors to current Node 24 majors, resolving the deprecation warning on every CI run. - -Closes ITL-621 - -## Actions upgraded - -| Action | Old | New | -|---|---|---| -| `actions/dependency-review-action` | v4.9.0 | v5.0.0 | -| `actions/setup-python` | v5.6.0 | v7.0.0 | -| `codecov/codecov-action` | v5.5.5 | v7.0.0 | -| `actions/checkout` | v4.4.0 | v7.0.0 | -| `astral-sh/setup-uv` | v5.4.2 | v10.0.1 | - -## Breaking change audit - -| Action | Breaking changes crossed | Finding | -|---|---|---| -| `dependency-review-action` v4→v5 | Node 20→24 runtime only | ✅ No-op — `deny-licenses` list unchanged | -| `setup-python` v5→v7 | v6: Node 20→24; v7: `pip-install` input removed | ✅ No-op — we don't use `pip-install`; uv handles all installs | -| `codecov-action` v5→v7 | v6: Node 24; v7: GPG key rotation | ✅ No-op — we don't verify Codecov signatures | -| `checkout` v4→v7 | v5: Node 20→24; v6: creds in separate file; v7: fork-head block for `pull_request_target`/`workflow_run` | ✅ No-op — `release.yml` git operations unaffected by new cred location; we don't use `pull_request_target` | -| `setup-uv` v5→v10 | v6–v8: input/tag changes; v9: `prune-cache` default flips true→false; v10: cache disabled for `pull_request_target`/`workflow_run` | ✅ Mostly no-op — we use none of the changed inputs; v10 cache change doesn't apply; `prune-cache` flip accepted (see below) | - -## prune-cache note - -`setup-uv` v9 flipped `prune-cache` from `true` to `false`. Decision: accept the new default and monitor cache growth. A follow-up issue has been created to revisit if storage becomes a concern. - -## Commit structure - -Five commits, one per action, ordered risk-ascending — bisectable if any action changes behaviour subtly. -EOF -)" -``` - ---- - -## Self-review checklist - -After all tasks complete, verify against spec acceptance criteria: - -- [ ] All five actions on current major, SHA-pinned with accurate `# vX.Y.Z` comments -- [ ] No `v4.4.0`, `v5.4.2`, `v5.6.0`, `v5.5.5`, or `v4.9.0` version comments remain -- [ ] Breaking change findings documented in PR body -- [ ] Follow-up Linear issue created for prune-cache monitoring -- [ ] PR targets `main` diff --git a/superpowers/plans/2026-08-20-itl-615-pollingsource-pipelinejob-async-delegation-fix.md b/superpowers/plans/2026-08-20-itl-615-pollingsource-pipelinejob-async-delegation-fix.md deleted file mode 100644 index 607d0064c..000000000 --- a/superpowers/plans/2026-08-20-itl-615-pollingsource-pipelinejob-async-delegation-fix.md +++ /dev/null @@ -1,327 +0,0 @@ -# ITL-615: PollingSource PipelineJob Async Delegation Fix — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix `SourceJobNode.async_iter_data()` to delegate to `bound_source.async_iter_data()` so that a `PollingSource` bound into a `PipelineJob` runs its full async polling loop under `AsyncPipelineOrchestrator`. - -**Architecture:** Add one 6-line method override to `SourceJobNode` in `source_node.py`. The method mirrors the existing delegation pattern used by `iter_data()`, `output_schema()`, and `as_table()`. Add an integration test file and a `DESIGN_ISSUES.md` entry. - -**Tech Stack:** Python 3.11+, asyncio, pytest-asyncio, pyarrow, orcapod internal classes. - ---- - -## File Map - -| Action | Path | What changes | -|--------|------|-------------| -| Modify | `src/orcapod/core/nodes/source_node.py` | Add `async_iter_data()` to `SourceJobNode` | -| Create | `tests/test_channels/test_polling_source_pipeline_integration.py` | Integration tests | -| Modify | `DESIGN_ISSUES.md` | Add bug entry | - ---- - -### Task 1: Add DESIGN_ISSUES.md entry - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Open DESIGN_ISSUES.md and add the entry** - - Add the following block immediately after the `## Cross-cutting` header section - (before the first `###` entry), or at the end of the file if no clear section - applies. Insert after the `## Cross-cutting` section: - - ```markdown - ## `src/orcapod/core/nodes/source_node.py` - - ### SJN1 — `SourceJobNode.async_iter_data()` wraps `iter_data()` synchronously, bypassing `PollingSource` polling loop - **Status:** in progress - **Severity:** high - **Issue:** ITL-615 - - `SourceJobNode` does not override `async_iter_data()`. The base-class - `SourceNodeBase.async_iter_data()` wraps `self.iter_data()` (sync) as an async - generator, so `bound_source.async_iter_data()` is never called. For - `PollingSource`, `iter_data()` returns a static single-batch snapshot — - the async polling loop is never started. - - **Fix:** Add `SourceJobNode.async_iter_data()` that delegates to - `self._bound_source.async_iter_data()`, consistent with the existing delegation - pattern of `iter_data()`, `output_schema()`, and `as_table()`. - ``` - -- [ ] **Step 2: Commit** - - ```bash - git add DESIGN_ISSUES.md - git commit -m "docs(design-issues): log ITL-615 SourceJobNode async_iter_data bypass bug" - ``` - ---- - -### Task 2: Write the failing integration test - -**Files:** -- Create: `tests/test_channels/test_polling_source_pipeline_integration.py` - -- [ ] **Step 1: Create the test file** - - Create `tests/test_channels/test_polling_source_pipeline_integration.py` with the - following content: - - ```python - """Integration tests: PollingSource bound into PipelineJob + AsyncPipelineOrchestrator. - - Verifies that the async polling loop runs (not just a static snapshot) when - a PollingSource is bound as the data source for a PipelineJob and executed - via the async orchestrator. - """ - from __future__ import annotations - - import pytest - import pyarrow as pa - - from orcapod import function_pod - from orcapod.core.sources.polling_source import PollingSource - from orcapod.databases import InMemoryArrowDatabase - from orcapod.errors import UnboundSourceError - from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator - from orcapod.pipeline.job import PipelineJob - from orcapod.core.nodes.source_node import SourceJobNode - from orcapod.types import Cursor, PollingConfig, Schema - - - # --------------------------------------------------------------------------- - # Shared fake impl — two-batch DynamicSourceProtocol - # --------------------------------------------------------------------------- - - class _TwoBatchImpl: - """Serves exactly 2 batches then reports no new data. - - batch 0: id=1, val=10 - batch 1: id=2, val=20 - """ - - _BATCHES = [ - {"id": pa.array([1], type=pa.int64()), "val": pa.array([10], type=pa.int64())}, - {"id": pa.array([2], type=pa.int64()), "val": pa.array([20], type=pa.int64())}, - ] - - def identity(self): - return "_TwoBatchImpl" - - def to_config(self): - return None - - @classmethod - def from_config(cls, config): - return cls() - - def schema(self): - return Schema({"id": int, "val": int}) - - async def poll(self, cursor=None): - idx = cursor.value if cursor is not None else 0 - return idx < len(self._BATCHES) - - async def fetch(self, cursor=None): - idx = cursor.value if cursor is not None else 0 - if idx >= len(self._BATCHES): - return Cursor(value=idx), {} - return Cursor(value=idx + 1), self._BATCHES[idx] - - async def close(self): - pass - - - # --------------------------------------------------------------------------- - # Downstream function pod used by integration tests - # --------------------------------------------------------------------------- - - @function_pod(output_keys="doubled") - def _double_val(val: int) -> int: - """Doubles the incoming value — used to produce a queryable DB record.""" - return val * 2 - - - # --------------------------------------------------------------------------- - # Tests - # --------------------------------------------------------------------------- - - class TestPollingSourcePipelineJobIntegration: - @pytest.mark.asyncio - async def test_async_orchestrator_runs_polling_loop(self): - """PollingSource bound to PipelineJob runs the full async polling loop. - - A static snapshot would process only the first batch (id=1, val=10) - and produce 1 DB record. The polling loop must process both batches and - produce 2 DB records. - """ - src = PollingSource( - _TwoBatchImpl(), - tag_columns="id", - polling_config=PollingConfig( - interval=0.05, - duration=0.5, - max_missed_intervals=50, - ), - source_id="two_batch_src", - ) - - store = InMemoryArrowDatabase() - job = PipelineJob(name="polling_integration_test", store=store) - with job: - _double_val.pod(src, label="doubled") - - await AsyncPipelineOrchestrator().run_async(job.dag) - job.store.at(*job._name).flush() - job.store.at(*job._name).at("_result").flush() - - records = job.nodes["doubled"].get_all_records() - assert records is not None - # Both batches must be processed — static snapshot yields only 1 row - assert records.num_rows == 2, ( - f"Expected 2 rows (one per batch), got {records.num_rows}. " - "This means only the static snapshot ran, not the polling loop." - ) - vals = sorted(records.column("val").to_pylist()) - assert vals == [10, 20] - doubled = sorted(records.column("doubled").to_pylist()) - assert doubled == [20, 40] - - @pytest.mark.asyncio - async def test_unbound_source_job_node_async_iter_raises(self): - """Unbound SourceJobNode.async_iter_data() raises UnboundSourceError.""" - node = SourceJobNode( - name="unbound", - tag_schema=Schema({"id": int}), - data_schema=Schema({"val": int}), - bound_source=None, - ) - - with pytest.raises(UnboundSourceError): - async for _ in node.async_iter_data(): - pass - ``` - -- [ ] **Step 2: Run tests to confirm both fail** - - ```bash - uv run pytest tests/test_channels/test_polling_source_pipeline_integration.py -v - ``` - - Expected output (both tests fail before the fix): - - `test_async_orchestrator_runs_polling_loop` — FAIL with `AssertionError: Expected 2 rows (one per batch), got 1` - - `test_unbound_source_job_node_async_iter_raises` — FAIL (no `UnboundSourceError` is raised because the base class iterates `iter_data()` which raises it at a different point, OR it passes for the wrong reason — either way the behavior before the fix needs verification) - - Note: The second test (`test_unbound_source_job_node_async_iter_raises`) may already pass - if `iter_data()` raises `UnboundSourceError` synchronously when iterated in the base-class - wrapper. That's acceptable — it becomes a regression guard. - -- [ ] **Step 3: Commit the failing test** - - ```bash - git add tests/test_channels/test_polling_source_pipeline_integration.py - git commit -m "test(channels): add failing integration test for ITL-615 PollingSource PipelineJob polling loop" - ``` - ---- - -### Task 3: Implement the fix - -**Files:** -- Modify: `src/orcapod/core/nodes/source_node.py` - -- [ ] **Step 1: Locate the insertion point** - - Open `src/orcapod/core/nodes/source_node.py`. Find the `SourceJobNode` class. - Find the `iter_data()` method (around line 556). The new `async_iter_data()` method - goes immediately after `iter_data()`. - -- [ ] **Step 2: Add the `async_iter_data()` override** - - Insert the following method after `iter_data()` and before `output_schema()` in - `SourceJobNode`: - - ```python - async def async_iter_data(self): - """Delegate to ``bound_source.async_iter_data()`` when bound. - - Overrides ``SourceNodeBase.async_iter_data()`` to route through the - bound source's own async generator instead of wrapping ``iter_data()`` - synchronously. This ensures that dynamic sources such as - ``PollingSource`` run their async polling loop rather than returning - a static snapshot. - - Raises: - UnboundSourceError: When no concrete source is attached. - """ - if self._bound_source is None: - raise UnboundSourceError( - f"SourceJobNode '{self._name}' has no concrete source bound. " - "Call job.bind(sources={'': source}) before running." - ) - async for pair in self._bound_source.async_iter_data(): - yield pair - ``` - - After this edit, `SourceJobNode` has these delegation methods (all consistent): - - | Method | Delegates to | - |--------|-------------| - | `iter_data()` | `self._bound_source.iter_data()` | - | `async_iter_data()` | `self._bound_source.async_iter_data()` ← new | - | `output_schema()` | `self._bound_source.output_schema()` | - | `as_table()` | `self._bound_source.as_table()` | - -- [ ] **Step 3: Run the integration tests — both should now pass** - - ```bash - uv run pytest tests/test_channels/test_polling_source_pipeline_integration.py -v - ``` - - Expected output: - ``` - tests/test_channels/test_polling_source_pipeline_integration.py::TestPollingSourcePipelineJobIntegration::test_async_orchestrator_runs_polling_loop PASSED - tests/test_channels/test_polling_source_pipeline_integration.py::TestPollingSourcePipelineJobIntegration::test_unbound_source_job_node_async_iter_raises PASSED - ``` - -- [ ] **Step 4: Run the full test suite to verify no regressions** - - ```bash - uv run pytest -m "not postgres" -x -q 2>&1 | tail -20 - ``` - - Expected: all existing tests pass. Pay special attention to: - - `tests/test_channels/test_polling_source.py` - - `tests/test_channels/test_pipeline_async_integration.py` - - `tests/test_pipeline/test_pipeline_job.py` - - `tests/test_pipeline/test_orchestrator.py` - -- [ ] **Step 5: Update DESIGN_ISSUES.md entry status** - - Find the `SJN1` entry added in Task 1 and change `**Status:** in progress` to - `**Status:** resolved`. Add a Fix note: - - ```markdown - **Fix:** Added `SourceJobNode.async_iter_data()` in `source_node.py` that delegates - to `self._bound_source.async_iter_data()`, consistent with the existing delegation - pattern of `iter_data()`, `output_schema()`, and `as_table()`. Added integration - test `test_polling_source_pipeline_integration.py`. - ``` - -- [ ] **Step 6: Commit** - - ```bash - git add src/orcapod/core/nodes/source_node.py DESIGN_ISSUES.md - git commit -m "fix(source-node): delegate async_iter_data to bound_source so PollingSource runs polling loop in PipelineJob (ITL-615)" - ``` - ---- - -## Self-Review Checklist - -- [x] Spec coverage: `SourceJobNode.async_iter_data()` fix → Task 3 ✓. Integration test → Task 2 ✓. DESIGN_ISSUES.md → Task 1 ✓. -- [x] No placeholders: All code blocks are complete, no TODOs. -- [x] Type consistency: `UnboundSourceError` is imported in the test (line 10). The method signature matches `SourceNodeBase.async_iter_data()` (returns an async generator). -- [x] The `_TwoBatchImpl` class satisfies `DynamicSourceProtocol` (has `identity`, `to_config`, `from_config`, `schema`, `poll`, `fetch`, `close`). diff --git a/superpowers/plans/2026-08-22-itl-627-list-extension-metadata.md b/superpowers/plans/2026-08-22-itl-627-list-extension-metadata.md deleted file mode 100644 index f5d037b4e..000000000 --- a/superpowers/plans/2026-08-22-itl-627-list-extension-metadata.md +++ /dev/null @@ -1,898 +0,0 @@ -# ITL-627: List-Extension Metadata — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix three defects where `ListLogicalType`-backed extension columns are mishandled by Join, the semantic hashing visitor, and MergeJoin. - -**Architecture:** Three independent fixes in three files (`list_logical_type_factory.py`, `visitors.py`, `merge_join.py`) plus test coverage in a new `tests/test_logical_types/` directory and additions to existing test files. Each task is independently committable. - -**Tech Stack:** PyArrow extension types, Polars C data interface, Python `typing` module generics, `get_default_context()` for type converter access. - -**Spec:** `superpowers/specs/2026-08-22-itl-627-list-extension-metadata.md` - ---- - -## File Map - -| File | Action | Purpose | -|---|---|---| -| `tests/test_logical_types/__init__.py` | Create | Make new package importable | -| `tests/test_logical_types/test_list_logical_type.py` | Create | Unit regression tests for Fix 1 | -| `src/orcapod/logical_types/list_logical_type_factory.py` | Modify line 151 | Fix 1: pass `metadata=` | -| `tests/test_core/operators/test_operators.py` | Modify | Integration test for Fix 1 via Join | -| `tests/test_core/operators/test_merge_join.py` | Modify | Integration test for Fix 1 via MergeJoin + Fix 3 tests | -| `src/orcapod/hashing/visitors.py` | Modify `visit_extension` | Fix 2: delegate list-backed extension to `_visit_list_elements` | -| `tests/test_hashing/test_extension_type_hashing.py` | Modify | Fix 2 regression + symmetry tests | -| `src/orcapod/core/operators/merge_join.py` | Modify `binary_static_process` | Fix 3: produce extension-typed merged arrays | - ---- - -## Task 1: Fix 1 — pass metadata to `make_polars_extension_type` - -**Files:** -- Create: `tests/test_logical_types/__init__.py` -- Create: `tests/test_logical_types/test_list_logical_type.py` -- Modify: `src/orcapod/logical_types/list_logical_type_factory.py:149-155` - -- [ ] **Step 1.1: Create the new test package** - -```bash -touch tests/test_logical_types/__init__.py -``` - -- [ ] **Step 1.2: Write the failing unit tests** - -Create `tests/test_logical_types/test_list_logical_type.py`: - -```python -"""Regression tests for ListLogicalType Polars extension type metadata. - -Defect 1 (ITL-627): get_polars_extension_type() was called without passing -metadata= to make_polars_extension_type, so ext_metadata() returned None. -Polars exported b'' on to_arrow(), which _deserialize rejected with ValueError. -""" - -from __future__ import annotations - -import json - -import pytest - -from orcapod.logical_types.builtin_logical_types import LogicalPath -from orcapod.logical_types.list_logical_type_factory import ListLogicalType - - -class TestListLogicalTypePolarsMetadata: - def test_list_polars_ext_carries_metadata(self): - """get_polars_extension_type() for list[Path] must carry JSON metadata. - - With the buggy code (metadata= not passed), ext_metadata() returns None. - Polars then exports b'' on to_arrow(), causing _deserialize to raise - ValueError because b'' != the expected JSON bytes. - """ - lt = ListLogicalType(LogicalPath(), is_set=False) - polars_ext = lt.get_polars_extension_type() - - meta = polars_ext.ext_metadata() - assert meta is not None, ( - "ext_metadata() returned None — metadata= was not passed to " - "make_polars_extension_type. This causes the Polars→Arrow round-trip to fail." - ) - parsed = json.loads(meta) - assert parsed["category"] == "list" - assert parsed["element_ext_name"] == "orcapod.path" - - def test_set_polars_ext_carries_metadata(self): - """Same one-line fix covers set[T] (identical code path, is_set=True).""" - lt = ListLogicalType(LogicalPath(), is_set=True) - polars_ext = lt.get_polars_extension_type() - - meta = polars_ext.ext_metadata() - assert meta is not None - parsed = json.loads(meta) - assert parsed["category"] == "set" - assert parsed["element_ext_name"] == "orcapod.path" - - def test_polars_to_arrow_round_trip_preserves_extension_type(self): - """Full Polars round-trip must not raise and must preserve the extension type. - - pl.DataFrame(table).to_arrow() calls _deserialize; without the fix it - receives b'' and raises ValueError. - """ - import pyarrow as pa - import polars as pl - - lt = ListLogicalType(LogicalPath(), is_set=False) - ext_type = lt.get_arrow_extension_type() - - storage = pa.array([["/a.txt", "/b.txt"]], type=pa.large_list(pa.large_string())) - ext_array = pa.ExtensionArray.from_storage(ext_type, storage) - table = pa.table({"paths": ext_array}) - - # Must not raise ValueError from _deserialize - result = pl.DataFrame(table).to_arrow() - - assert isinstance(result.schema.field("paths").type, pa.ExtensionType) - assert result.schema.field("paths").type.extension_name == "list[orcapod.path]" -``` - -- [ ] **Step 1.3: Run to verify tests fail** - -```bash -uv run pytest tests/test_logical_types/test_list_logical_type.py -v -``` - -Expected: 3 FAILED — `ext_metadata()` returns `None`, round-trip raises `ValueError`. - -- [ ] **Step 1.4: Apply Fix 1** - -In `src/orcapod/logical_types/list_logical_type_factory.py`, modify `get_polars_extension_type` (lines 149–155): - -```python - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type for this list/set logical type. - - Returns: - A cached ``pl.BaseExtension`` instance registered under the logical - type name. - """ - if self._polars_ext is None: - polars_ext_class = make_polars_extension_type( - self._logical_type_name, - self._storage_type, - metadata=self._metadata_bytes.decode("utf-8"), - ) - self._polars_ext = polars_ext_class() - return self._polars_ext -``` - -The only change is adding `metadata=self._metadata_bytes.decode("utf-8")` on the line after `self._storage_type,`. - -- [ ] **Step 1.5: Run to verify tests pass** - -```bash -uv run pytest tests/test_logical_types/test_list_logical_type.py -v -``` - -Expected: 3 PASSED. - -- [ ] **Step 1.6: Commit** - -```bash -git add tests/test_logical_types/__init__.py \ - tests/test_logical_types/test_list_logical_type.py \ - src/orcapod/logical_types/list_logical_type_factory.py -git commit -m "fix(logical_types): pass metadata to make_polars_extension_type in ListLogicalType - -ListLogicalType.get_polars_extension_type() was not passing the JSON metadata -bytes to make_polars_extension_type, so ext_metadata() returned None. Polars -exported b'' on to_arrow(), which _deserialize rejected with ValueError during -the Join/MergeJoin Polars round-trip. One-line fix covers both list[T] and set[T]. - -Fixes ITL-627 (Defect 1)." -``` - ---- - -## Task 2: Defect 1 integration tests — Join and MergeJoin preserve list extension column - -**Files:** -- Modify: `tests/test_core/operators/test_operators.py` -- Modify: `tests/test_core/operators/test_merge_join.py` - -- [ ] **Step 2.1: Add the Join integration test** - -In `tests/test_core/operators/test_operators.py`, add a new test class after `TestJoinBehavior`: - -```python -class TestJoinWithListExtensionColumn: - """Regression tests for ITL-627 Defect 1: Join Polars round-trip with list extension columns.""" - - def test_join_preserves_list_extension_column(self): - """Join must not raise and must preserve extension. - - Before Fix 1, df.to_arrow() inside static_process called _deserialize - with b'' (no metadata), raising ValueError. - """ - import pyarrow as pa - from orcapod.logical_types.builtin_logical_types import LogicalPath - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - - lt = ListLogicalType(LogicalPath(), is_set=False) - ext_type = lt.get_arrow_extension_type() - - storage = pa.array( - [["/a.txt", "/b.txt"], ["/c.txt"]], - type=pa.large_list(pa.large_string()), - ) - ext_array = pa.ExtensionArray.from_storage(ext_type, storage) - left_table = pa.table({ - "animal": pa.array(["cat", "dog"], type=pa.large_string()), - "paths": ext_array, - }) - left_stream = ArrowTableStream(left_table, tag_columns=["animal"]) - - right_table = pa.table({ - "animal": pa.array(["cat", "dog"], type=pa.large_string()), - "speed": pa.array([30.0, 45.0], type=pa.float64()), - }) - right_stream = ArrowTableStream(right_table, tag_columns=["animal"]) - - op = Join() - result = op.static_process(left_stream, right_stream) # must not raise - out_table = result.as_table() - - paths_type = out_table.schema.field("paths").type - assert isinstance(paths_type, pa.ExtensionType), ( - f"'paths' column must remain an extension type, got {paths_type}" - ) - assert paths_type.extension_name == "list[orcapod.path]" -``` - -- [ ] **Step 2.2: Add the MergeJoin round-trip test (non-colliding list column)** - -In `tests/test_core/operators/test_merge_join.py`, add a new test class: - -```python -class TestMergeJoinWithListExtensionColumn: - """Regression for ITL-627 Defect 1: MergeJoin Polars round-trip with list extension columns.""" - - def test_merge_join_preserves_non_colliding_list_extension_column(self): - """MergeJoin must not raise and must preserve extension - for a non-colliding list[Path] data column. - - MergeJoin also does a Polars round-trip; without Fix 1 it raises ValueError. - """ - import pyarrow as pa - from orcapod.logical_types.builtin_logical_types import LogicalPath - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - - lt = ListLogicalType(LogicalPath(), is_set=False) - ext_type = lt.get_arrow_extension_type() - - storage = pa.array( - [["/a.txt", "/b.txt"], ["/c.txt"]], - type=pa.large_list(pa.large_string()), - ) - ext_array = pa.ExtensionArray.from_storage(ext_type, storage) - - # Left stream: list[Path] data column (non-colliding) + shared tag - left_table = pa.table({ - "id": pa.array([1, 2], type=pa.int64()), - "paths": ext_array, - }) - left_stream = ArrowTableStream(left_table, tag_columns=["id"]) - - # Right stream: different non-colliding data column + same tag - right_table = pa.table({ - "id": pa.array([1, 2], type=pa.int64()), - "score": pa.array([10.0, 20.0], type=pa.float64()), - }) - right_stream = ArrowTableStream(right_table, tag_columns=["id"]) - - result = MergeJoin().static_process(left_stream, right_stream) # must not raise - out_table = result.as_table() - - paths_type = out_table.schema.field("paths").type - assert isinstance(paths_type, pa.ExtensionType), ( - f"'paths' column must remain an extension type, got {paths_type}" - ) - assert paths_type.extension_name == "list[orcapod.path]" -``` - -- [ ] **Step 2.3: Run both new tests to verify they pass (Fix 1 already applied)** - -```bash -uv run pytest tests/test_core/operators/test_operators.py::TestJoinWithListExtensionColumn \ - tests/test_core/operators/test_merge_join.py::TestMergeJoinWithListExtensionColumn \ - -v -``` - -Expected: 2 PASSED (Fix 1 from Task 1 already resolves these). - -- [ ] **Step 2.4: Commit** - -```bash -git add tests/test_core/operators/test_operators.py \ - tests/test_core/operators/test_merge_join.py -git commit -m "test(operators): add regression tests for list extension column round-trip - -Exercises the full Join and MergeJoin Polars round-trip with list[Path] extension -columns, confirming Fix 1 (ITL-627 Defect 1) at the operator integration level." -``` - ---- - -## Task 3: Fix 2 — semantic hashing for list-backed extension columns - -**Files:** -- Modify: `src/orcapod/hashing/visitors.py` — `SemanticHashingVisitor.visit_extension` -- Modify: `tests/test_hashing/test_extension_type_hashing.py` - -- [ ] **Step 3.1: Write the failing hashing tests** - -Add a new test class `TestListExtensionHashing` to `tests/test_hashing/test_extension_type_hashing.py`: - -```python -class TestListExtensionHashing: - """Regression and contract tests for ITL-627 Defect 2. - - Before Fix 2, visit_extension short-circuited on `not isinstance(python_type, type)` - for list[File] (a GenericAlias), returning the extension type unchanged — raw JSON - path strings were hashed instead of file contents. - """ - - def _make_list_file_ext_type(self, ctx): - """Return the extension Arrow type via the type converter.""" - from orcapod.logical_types.file_type import File - ctx.type_converter.register_python_class(list[File]) - return ctx.type_converter.python_type_to_arrow_type(list[File]) - - def _make_scalar_file_ext_type(self, ctx): - """Return the extension Arrow type.""" - from orcapod.logical_types.file_type import File - return ctx.type_converter.register_python_class(File) - - def _file_storage(self, ctx, path): - """Return the large_string storage value for a File.""" - from orcapod.logical_types.file_type import File - return ctx.type_converter.python_to_storage(File(path), File) - - def test_list_file_extension_hashed_to_list_of_large_binary(self, ctx, tmp_path): - """visit_extension for extension must return - (large_list(large_binary), [bytes, bytes]) — not the extension type unchanged. - - With the buggy code the isinstance guard exits immediately, returning - (extension_type, storage_value). This assertion on new_type would fail. - """ - f0 = tmp_path / "f0.txt"; f0.write_text("alpha") - f1 = tmp_path / "f1.txt"; f1.write_text("beta") - - list_ext_type = self._make_list_file_ext_type(ctx) - s0 = self._file_storage(ctx, f0) - s1 = self._file_storage(ctx, f1) - storage_value = [s0, s1] - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(list_ext_type, storage_value) - - import pyarrow as pa - assert new_type == pa.large_list(pa.large_binary()), ( - f"Expected large_list(large_binary), got {new_type}. " - "Buggy code returns the extension type unchanged." - ) - assert isinstance(new_data, list) - assert len(new_data) == 2 - assert all(isinstance(b, bytes) for b in new_data) - - def test_list_file_extension_is_hash_of_file_content_hashes(self, ctx, tmp_path): - """Each element of the list result equals the scalar visit result for the same file. - - Contract: visit(list_ext, [s0, s1])[1][i] == visit(scalar_ext, si)[1] - This is the symmetry invariant — list[File] and scalar File hash identically - per element. Starfix then sees an ordered list of content-hash tokens. - """ - f0 = tmp_path / "contract0.txt"; f0.write_text("content zero") - f1 = tmp_path / "contract1.txt"; f1.write_text("content one") - - scalar_ext_type = self._make_scalar_file_ext_type(ctx) - list_ext_type = self._make_list_file_ext_type(ctx) - s0 = self._file_storage(ctx, f0) - s1 = self._file_storage(ctx, f1) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - - # Scalar hashes - _, h0_bytes = visitor.visit(scalar_ext_type, s0) - _, h1_bytes = visitor.visit(scalar_ext_type, s1) - - # List hash - _, list_result = visitor.visit(list_ext_type, [s0, s1]) - - assert list_result[0] == h0_bytes, ( - "Element 0 of list result must equal the scalar hash of file 0" - ) - assert list_result[1] == h1_bytes, ( - "Element 1 of list result must equal the scalar hash of file 1" - ) - - def test_list_file_extension_content_change_changes_hash(self, ctx, tmp_path): - """Changing file content changes the per-element hash.""" - f = tmp_path / "mutable.txt" - f.write_text("v1") - - list_ext_type = self._make_list_file_ext_type(ctx) - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - - s_v1 = self._file_storage(ctx, f) - _, result_v1 = visitor.visit(list_ext_type, [s_v1]) - r0_v1 = result_v1[0] - - f.write_text("v2") - from orcapod.logical_types.file_type import File - s_v2 = ctx.type_converter.python_to_storage(File(f), File) - _, result_v2 = visitor.visit(list_ext_type, [s_v2]) - r0_v2 = result_v2[0] - - assert r0_v1 != r0_v2, "Content change must change the per-element hash" - - def test_list_file_extension_same_content_same_hash(self, ctx, tmp_path): - """Two files with identical content produce identical per-element hashes.""" - fa = tmp_path / "a.txt"; fa.write_text("identical") - fb = tmp_path / "b.txt"; fb.write_text("identical") - - list_ext_type = self._make_list_file_ext_type(ctx) - sa = self._file_storage(ctx, fa) - sb = self._file_storage(ctx, fb) - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - _, result_a = visitor.visit(list_ext_type, [sa]) - _, result_b = visitor.visit(list_ext_type, [sb]) - - assert result_a[0] == result_b[0], ( - "Same content at different paths must produce the same hash" - ) - - def test_list_file_extension_passthrough_when_no_handler(self, ctx, tmp_path): - """When the registry has no FileHandler, visit_extension must passthrough.""" - from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeHandlerRegistry - from orcapod.hashing.semantic_hashing.semantic_hasher import SemanticAwarePythonHasher - - empty_registry = PythonTypeHandlerRegistry() - stripped_hasher = SemanticAwarePythonHasher( - hasher_id="test_v0", - type_handler_registry=empty_registry, - ) - - f = tmp_path / "f.txt"; f.write_text("test") - list_ext_type = self._make_list_file_ext_type(ctx) - storage_value = [self._file_storage(ctx, f)] - - visitor = SemanticHashingVisitor(ctx.type_converter, stripped_hasher) - new_type, new_data = visitor.visit(list_ext_type, storage_value) - - assert new_type == list_ext_type, "Must passthrough extension type when no handler" - assert new_data == storage_value, "Must passthrough storage value when no handler" - - def test_list_path_extension_passthrough(self, ctx, tmp_path): - """extension must passthrough — Path has no content handler.""" - from orcapod.logical_types.builtin_logical_types import LogicalPath - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - import pyarrow as pa - from pathlib import Path - - lt = ListLogicalType(LogicalPath(), is_set=False) - list_ext_type = lt.get_arrow_extension_type() - storage_value = ["/a.txt", "/b.txt"] - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(list_ext_type, storage_value) - - assert new_type == list_ext_type, "Path has no handler — must passthrough" - assert new_data == storage_value - - def test_set_file_extension_hashed_to_list_of_large_binary(self, ctx, tmp_path): - """set[File] (extension) also hashes per element. - - get_origin(set[File]) is `set`, covered by `in (list, set)` in Fix 2. - """ - from orcapod.logical_types.file_type import File - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - from orcapod.logical_types.file_type import LogicalFile - import pyarrow as pa - - f0 = tmp_path / "s0.txt"; f0.write_text("set alpha") - f1 = tmp_path / "s1.txt"; f1.write_text("set beta") - - lt = ListLogicalType(LogicalFile(), is_set=True) - set_ext_type = lt.get_arrow_extension_type() - s0 = self._file_storage(ctx, f0) - s1 = self._file_storage(ctx, f1) - storage_value = [s0, s1] - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(set_ext_type, storage_value) - - assert new_type == pa.large_list(pa.large_binary()) - assert isinstance(new_data, list) - assert len(new_data) == 2 - assert all(isinstance(b, bytes) for b in new_data) - - def test_list_list_file_extension_hashed_recursively(self, ctx, tmp_path): - """extension recurses: each inner list becomes large_list(large_binary). - - Fix 2 recurses naturally: outer visit_extension delegates to _visit_list_elements - with virtual_type=large_list(extension), which calls - visit(extension, inner_list) for each element, which - recurses back into visit_extension. - """ - from orcapod.logical_types.file_type import File, LogicalFile - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - import pyarrow as pa - - f0 = tmp_path / "n0.txt"; f0.write_text("nested zero") - f1 = tmp_path / "n1.txt"; f1.write_text("nested one") - f2 = tmp_path / "n2.txt"; f2.write_text("nested two") - - inner_lt = ListLogicalType(LogicalFile(), is_set=False) - outer_lt = ListLogicalType(inner_lt, is_set=False) - outer_ext_type = outer_lt.get_arrow_extension_type() - - s0 = self._file_storage(ctx, f0) - s1 = self._file_storage(ctx, f1) - s2 = self._file_storage(ctx, f2) - # One row: [[s0, s1], [s2]] - storage_value = [[s0, s1], [s2]] - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(outer_ext_type, storage_value) - - assert new_type == pa.large_list(pa.large_list(pa.large_binary())), ( - f"Expected large_list(large_list(large_binary)), got {new_type}" - ) - assert len(new_data) == 2 - assert len(new_data[0]) == 2 # two files in first inner list - assert len(new_data[1]) == 1 # one file in second inner list - assert all(isinstance(b, bytes) for b in new_data[0]) - assert all(isinstance(b, bytes) for b in new_data[1]) - - def test_dataclass_with_list_file_field_hashed(self, ctx, tmp_path): - """A Dataclass stored as a struct whose field is list[File] must hash per element. - - visit_struct recurses into fields via visit(field_type, field_data). - The list[File] field has type extension, so visit - dispatches to visit_extension, which Fix 2 handles correctly. - """ - from dataclasses import dataclass - from orcapod.logical_types.file_type import File - import pyarrow as pa - - @dataclass - class FileBundle: - name: str - files: list[File] - - # Register the dataclass with the type converter - ctx.type_converter.register_python_class(FileBundle) - arrow_type = ctx.type_converter.python_type_to_arrow_type(FileBundle) - - f0 = tmp_path / "dc0.txt"; f0.write_text("dc alpha") - f1 = tmp_path / "dc1.txt"; f1.write_text("dc beta") - - s0 = ctx.type_converter.python_to_storage(File(f0), File) - s1 = ctx.type_converter.python_to_storage(File(f1), File) - storage_value = {"name": "bundle", "files": [s0, s1]} - - visitor = SemanticHashingVisitor(ctx.type_converter, ctx.semantic_hasher) - new_type, new_data = visitor.visit(arrow_type, storage_value) - - # The `files` field should be hashed to large_list(large_binary) - files_field_type = new_type.field("files").type - assert files_field_type == pa.large_list(pa.large_binary()), ( - f"files field must be large_list(large_binary), got {files_field_type}" - ) - files_hashes = new_data["files"] - assert len(files_hashes) == 2 - assert all(isinstance(b, bytes) for b in files_hashes) -``` - -- [ ] **Step 3.2: Run to verify all new tests fail** - -```bash -uv run pytest tests/test_hashing/test_extension_type_hashing.py::TestListExtensionHashing -v -``` - -Expected: All FAILED — the `isinstance(python_type, type)` guard short-circuits for `list[File]`. - -- [ ] **Step 3.3: Apply Fix 2** - -In `src/orcapod/hashing/visitors.py`, replace the full `visit_extension` method of `SemanticHashingVisitor` (lines 196–231): - -```python - def visit_extension( - self, - extension_type: "pa.ExtensionType", - storage_value: Any, - ) -> tuple["pa.DataType", Any]: - """Hash an extension type value to pa.large_binary(), or passthrough. - - For list-backed extension types (e.g. ``extension``), - delegates to ``_visit_list_elements`` with a virtual - ``large_list(elem_ext_type)`` so that each element is hashed identically - to the scalar ``visit_extension`` path. This covers ``list[T]``, - ``set[T]``, and arbitrary nesting depth via recursion. - """ - if storage_value is None: - return extension_type, None - - # Resolve extension type → Python type. - python_type = self._type_converter.arrow_type_to_python_type(extension_type) - - # Detect list-backed extension types: extension, - # extension, etc. list[File] is a types.GenericAlias - # (not isinstance(..., type)), so the guard below would incorrectly skip it. - # We intercept here and delegate to _visit_list_elements with a virtual - # large_list(elem_ext_type) so each element goes through visit_extension. - if ( - typing.get_origin(python_type) in (list, set) - and pa.types.is_large_list(extension_type.storage_type) - ): - args = typing.get_args(python_type) - if args: - elem_python_type = args[0] - if ( - isinstance(elem_python_type, type) - and self._python_hasher.type_handler_registry.has_handler( - elem_python_type - ) - ): - elem_arrow_type = self._type_converter.python_type_to_arrow_type( - elem_python_type - ) - virtual_list_type = pa.large_list(elem_arrow_type) - return self._visit_list_elements(virtual_list_type, storage_value) - - # If the converter couldn't resolve to a concrete class, passthrough. - if python_type is typing.Any or not isinstance(python_type, type): - return extension_type, storage_value - - # Only hash if a semantic hasher is registered for this Python type. - if not self._python_hasher.type_handler_registry.has_handler(python_type): - return extension_type, storage_value - - # Convert storage value → Python object and hash it. - python_obj = self._type_converter.storage_to_python(storage_value, python_type) - content_hash = self._python_hasher.hash_object(python_obj) - - # Encode as binary: ":::" - # Dots in the extension name → colons (e.g. "orcapod.path" → "orcapod:path"). - # The "::" separator is unambiguous because to_prefixed_digest() uses only ":". - type_name = extension_type.extension_name.replace(".", ":") - hash_bytes = ( - type_name.encode("utf-8") - + b"::" - + content_hash.to_prefixed_digest() - ) - return pa.large_binary(), hash_bytes -``` - -- [ ] **Step 3.4: Run to verify new tests pass** - -```bash -uv run pytest tests/test_hashing/test_extension_type_hashing.py -v -``` - -Expected: All PASSED (new tests + pre-existing tests). - -- [ ] **Step 3.5: Commit** - -```bash -git add src/orcapod/hashing/visitors.py \ - tests/test_hashing/test_extension_type_hashing.py -git commit -m "fix(hashing): hash list-backed extension columns element-by-element - -SemanticHashingVisitor.visit_extension short-circuited on -\`not isinstance(python_type, type)\` for list[File] (a GenericAlias), returning -the extension type unchanged. Raw JSON path strings were hashed instead of file -contents. Fix detects list/set-backed extension types and delegates to -_visit_list_elements with a virtual large_list(elem_ext_type), producing -per-element content hashes identical to the scalar visit path. Recurses -naturally for nested list[list[T]]. - -Fixes ITL-627 (Defect 2)." -``` - ---- - -## Task 4: Fix 3 — MergeJoin produces extension-typed merged arrays - -**Files:** -- Modify: `src/orcapod/core/operators/merge_join.py` — `binary_static_process` -- Modify: `tests/test_core/operators/test_merge_join.py` - -- [ ] **Step 4.1: Write the failing MergeJoin tests** - -Add `TestMergeJoinLogicalTypeColumns` to `tests/test_core/operators/test_merge_join.py`: - -```python -class TestMergeJoinLogicalTypeColumns: - """Regression tests for ITL-627 Defect 3. - - Before Fix 3, pa.array(merged_vals) inferred the array type from raw storage - values, producing plain large_list(storage_type) — the extension wrapper was lost. - """ - - def test_merge_join_scalar_logical_type_column_yields_list_extension(self, tmp_path): - """Merging a File column must produce extension, not large_list. - - Before Fix 3: pa.array([[json1, json2]]) inferred large_list(large_string). - After Fix 3: pa.ExtensionArray.from_storage(list_file_ext, ...) gives the correct type. - """ - import pyarrow as pa - from orcapod.logical_types.file_type import File, LogicalFile - from orcapod.contexts import get_default_context - - f1 = tmp_path / "mj1.txt"; f1.write_text("merge left") - f2 = tmp_path / "mj2.txt"; f2.write_text("merge right") - - ctx = get_default_context() - scalar_lt = LogicalFile() - ext_type = scalar_lt.get_arrow_extension_type() - - s1 = ctx.type_converter.python_to_storage(File(f1), File) - s2 = ctx.type_converter.python_to_storage(File(f2), File) - - left_table = pa.table({ - "id": pa.array([1], type=pa.int64()), - "file": pa.ExtensionArray.from_storage( - ext_type, pa.array([s1], type=pa.large_string()) - ), - }) - right_table = pa.table({ - "id": pa.array([1], type=pa.int64()), - "file": pa.ExtensionArray.from_storage( - ext_type, pa.array([s2], type=pa.large_string()) - ), - }) - left_stream = ArrowTableStream(left_table, tag_columns=["id"]) - right_stream = ArrowTableStream(right_table, tag_columns=["id"]) - - result = MergeJoin().static_process(left_stream, right_stream) - out_table = result.as_table() - - file_type = out_table.schema.field("file").type - assert isinstance(file_type, pa.ExtensionType), ( - f"'file' column must be extension, got {file_type}. " - "Buggy code produces plain large_list(large_string)." - ) - assert file_type.extension_name == "list[orcapod.file]" - - # Values must be a list of two storage values - file_values = out_table.column("file").to_pylist() - assert len(file_values) == 1 # one row - assert len(file_values[0]) == 2 # two merged elements - - def test_merge_join_list_backed_column_yields_nested_list_extension(self, tmp_path): - """Merging a list[File] column must produce extension. - - Fix 3 handles this naturally: elem_python_type = list[File], - get_logical_type_for_python_type(list[list[File]]) = ListLogicalType(ListLogicalType(LogicalFile())). - """ - import pyarrow as pa - from orcapod.logical_types.file_type import File, LogicalFile - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - from orcapod.contexts import get_default_context - - f1 = tmp_path / "nl1.txt"; f1.write_text("nested left") - f2 = tmp_path / "nl2.txt"; f2.write_text("nested right") - - ctx = get_default_context() - inner_lt = ListLogicalType(LogicalFile(), is_set=False) - inner_ext_type = inner_lt.get_arrow_extension_type() - - s1 = ctx.type_converter.python_to_storage(File(f1), File) - s2 = ctx.type_converter.python_to_storage(File(f2), File) - - # Each row's "files" value is a list of one file-storage-value - left_storage = pa.array([[s1]], type=pa.large_list(pa.large_string())) - right_storage = pa.array([[s2]], type=pa.large_list(pa.large_string())) - - left_table = pa.table({ - "id": pa.array([1], type=pa.int64()), - "files": pa.ExtensionArray.from_storage(inner_ext_type, left_storage), - }) - right_table = pa.table({ - "id": pa.array([1], type=pa.int64()), - "files": pa.ExtensionArray.from_storage(inner_ext_type, right_storage), - }) - left_stream = ArrowTableStream(left_table, tag_columns=["id"]) - right_stream = ArrowTableStream(right_table, tag_columns=["id"]) - - result = MergeJoin().static_process(left_stream, right_stream) - out_table = result.as_table() - - files_type = out_table.schema.field("files").type - assert isinstance(files_type, pa.ExtensionType), ( - f"'files' column must be extension, got {files_type}" - ) - assert files_type.extension_name == "list[list[orcapod.file]]" - - # One row with two inner lists - files_values = out_table.column("files").to_pylist() - assert len(files_values) == 1 - assert len(files_values[0]) == 2 -``` - -- [ ] **Step 4.2: Run to verify tests fail** - -```bash -uv run pytest tests/test_core/operators/test_merge_join.py::TestMergeJoinLogicalTypeColumns -v -``` - -Expected: 2 FAILED — column type is plain `large_list(large_string)`, not the extension type. - -- [ ] **Step 4.3: Apply Fix 3** - -In `src/orcapod/core/operators/merge_join.py`, make two changes to `binary_static_process`: - -**Change A** — add the colliding column type snapshot just before the Polars join (after the `output_nullable` block, around line 206). Insert before the `COMMON_JOIN_KEY` block: - -```python - # Snapshot Arrow types of colliding columns BEFORE the Polars round-trip. - # The round-trip may strip or alter extension metadata; we need the original - # element type to reconstruct the correct list extension type after merging. - colliding_col_types: dict[str, "pa.DataType"] = { - col: left_table.schema.field(col).type - for col in colliding_keys - if col in left_table.schema.names - } -``` - -**Change B** — replace the `merged_array = pa.array(merged_vals)` line (around line 276) with: - -```python - elem_arrow_type = colliding_col_types.get(col) - if elem_arrow_type is not None and isinstance(elem_arrow_type, pa.ExtensionType): - from orcapod.contexts import get_default_context - tc = get_default_context().type_converter - elem_python_type = tc.arrow_type_to_python_type(elem_arrow_type) - list_lt = tc.get_logical_type_for_python_type(list[elem_python_type]) - if list_lt is not None: - list_ext_type = list_lt.get_arrow_extension_type() - storage_array = pa.array(merged_vals, type=list_ext_type.storage_type) - merged_array = pa.ExtensionArray.from_storage(list_ext_type, storage_array) - else: - merged_array = pa.array(merged_vals) - else: - merged_array = pa.array(merged_vals) -``` - -- [ ] **Step 4.4: Run to verify new tests pass** - -```bash -uv run pytest tests/test_core/operators/test_merge_join.py -v -``` - -Expected: All PASSED. - -- [ ] **Step 4.5: Run the full test suite to check for regressions** - -```bash -uv run pytest tests/ -v --tb=short -``` - -Expected: All PASSED. - -- [ ] **Step 4.6: Commit** - -```bash -git add src/orcapod/core/operators/merge_join.py \ - tests/test_core/operators/test_merge_join.py -git commit -m "fix(operators): MergeJoin produces extension-typed list when merging logical-type columns - -binary_static_process used pa.array(merged_vals) which inferred array type from -raw storage values, producing plain large_list(storage_type) and losing the -extension wrapper. Fix snapshots the element Arrow type before the Polars round-trip, -then builds the merged array as pa.ExtensionArray.from_storage(ListLogicalType, ...) -when the element is an extension type. Handles nested list[list[T]] naturally. - -Fixes ITL-627 (Defect 3)." -``` - ---- - -## Self-Review - -**Spec coverage:** -- Defect 1 fix ✓ (Task 1) + unit test ✓ + integration tests ✓ (Task 2) -- Defect 1 `test_list_logical_type_polars_ext_carries_metadata` ✓ -- Defect 2 fix ✓ (Task 3) + all 8 hashing tests ✓ -- Defect 3 fix ✓ (Task 4) + scalar and nested MergeJoin tests ✓ -- `set[File]` test ✓ -- `list[list[File]]` hashing test ✓ -- Dataclass with `list[File]` field test ✓ -- `list[File] × list[File]` MergeJoin test ✓ - -**No placeholders:** All test code and implementation code is complete. - -**Type consistency:** `list_lt`, `list_ext_type`, `elem_arrow_type` are consistent across Fix 3 Part A and Part B. diff --git a/superpowers/plans/2026-08-25-itl-616-polling-source-combine-content-hash-fix.md b/superpowers/plans/2026-08-25-itl-616-polling-source-combine-content-hash-fix.md deleted file mode 100644 index 91f133b54..000000000 --- a/superpowers/plans/2026-08-25-itl-616-polling-source-combine-content-hash-fix.md +++ /dev/null @@ -1,331 +0,0 @@ -# ITL-616: Fix `PollingSource._combine` `_content_hash` Leak — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix `PollingSource._combine` so that the synthetic `_content_hash` column is never stored in the accumulated stream, preventing a `SchemaInconsistencyError` on the third accumulating fetch. - -**Architecture:** Add a module-level `_STREAM_COMBINE_COLUMNS = ColumnConfig(system_tags=True, source=True, context=True)` constant to `polling_source.py` and replace the two `as_table(all_info=True)` calls in `_combine` with `as_table(columns=_STREAM_COMBINE_COLUMNS)`. No other files change except `DESIGN_ISSUES.md` (new PS3 entry) and the test file (two new regression tests). - -**Tech Stack:** Python 3.11+, pytest, pytest-asyncio, pyarrow, orcapod internal classes (`PollingSource`, `ArrowTableStream`, `ColumnConfig`). - ---- - -## File Map - -| Action | Path | What changes | -|--------|------|-------------| -| Modify | `src/orcapod/core/sources/polling_source.py` | Add `_STREAM_COMBINE_COLUMNS` constant; replace `all_info=True` in `_combine` | -| Modify | `tests/test_channels/test_polling_source.py` | Add two regression tests to `TestPollingSourceSchemaValidation` | -| Modify | `DESIGN_ISSUES.md` | Add PS3 entry; update to `resolved` after fix | - ---- - -## Setup: Create the branch - -Before any task, create and check out the branch from `main`: - -```bash -cd /path/to/orcapod-python -git checkout main -git checkout -b eywalker/itl-616-pollingsource_combine-leaks-_content_hash-into-the-data -``` - -Verify: - -```bash -git branch --show-current -# eywalker/itl-616-pollingsource_combine-leaks-_content_hash-into-the-data -``` - ---- - -### Task 1: Add `DESIGN_ISSUES.md` PS3 entry - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Open `DESIGN_ISSUES.md` and insert the PS3 entry** - - Locate the block that ends with PS2 (search for `### PS2`). The section looks like this: - - ```markdown - ### PS2 — Concurrent iteration over a single `PollingSource` has unguarded cursor/stream mutation - **Status:** open - ... - - **Fix:** Guard updates to `_cursor` and `_accumulated_stream` with an `asyncio.Lock` ... - - --- - - ## `src/orcapod/core/nodes/function_node.py` - ``` - - Insert the following block **between** the `---` separator and the `## src/orcapod/core/nodes/function_node.py` header: - - ```markdown - ### PS3 — `_combine` leaks `_content_hash` into the data schema on the second accumulating fetch - **Status:** in progress - **Severity:** high - **Issue:** ITL-616 - - `_combine` calls `as_table(all_info=True)` on both streams before concatenating them. - `all_info=True` resolves to `ColumnConfig.all()`, which includes `content_hash=True`. - In `ArrowTableStream.as_table()`, `content_hash=True` dynamically appends a `_content_hash` - column to the output table. This is a synthetic column — computed on demand, not stored in - `ArrowTableStream._table`. - - `pa.concat_tables` then includes `_content_hash` in the combined table, which is passed - directly to `ArrowTableStream.__init__`. Since `_content_hash` has no recognized prefix - (`_tag::`, `_source_`, `_context_key`), it lands in `_data_columns` as if it were user data. - - On the next `_combine` call, `_validate_combining_schemas` compares: - - `existing.keys()` → includes `_content_hash` in data keys (baked in from previous combine) - - `new_stream.keys()` → no `_content_hash` (freshly built from raw fetched data) - - This raises `SchemaInconsistencyError`. A polling source emitting one new row per poll will - change its data schema on the second new-data poll and crash on the third. - - **Fix:** Replace `all_info=True` with a named module-level constant - `_STREAM_COMBINE_COLUMNS = ColumnConfig(system_tags=True, source=True, context=True)`. - `content_hash` is intentionally absent — it is a synthetic output column, never a stored one. - - --- - ``` - - Note: the `---` at the end of the PS3 entry replaces the existing `---` that was between PS2 and `## src/orcapod/core/nodes/function_node.py`. - -- [ ] **Step 2: Commit** - - ```bash - git add DESIGN_ISSUES.md - git commit -m "docs(design-issues): add PS3 entry for _combine content_hash leak (ITL-616)" - ``` - ---- - -### Task 2: Write the failing regression tests - -**Files:** -- Modify: `tests/test_channels/test_polling_source.py` - -The two tests go inside the existing `class TestPollingSourceSchemaValidation` (find it by searching for that class name). They must be placed **after** the last existing test in that class (`test_combining_column_set_mismatch_raises`). - -- [ ] **Step 1: Add the two tests to `TestPollingSourceSchemaValidation`** - - The test file already imports `pyarrow as pa`, `pytest`, `FakeDynamicSource`, `_batch`, `PollingSource`, `PollingConfig` — no new imports needed. - - Add these two methods at the end of `class TestPollingSourceSchemaValidation`: - - ```python - def test_sync_three_fetches_no_content_hash_leak(self): - """After 3 accumulating fetches (2 combines), data schema is stable and - contains no _content_hash column, and all rows are present. - - Regression test for ITL-616: _combine called as_table(all_info=True), - which injected the synthetic _content_hash column into the stored stream. - The second combine then raised SchemaInconsistencyError. - """ - fake = FakeDynamicSource( - batches=[_batch(1, 10), _batch(2, 20), _batch(3, 30)] - ) - src = PollingSource( - fake, tag_columns="id", polling_config=PollingConfig(interval=1.0) - ) - - rows1 = list(src.iter_data()) # fetch 1 — builds initial stream - rows2 = list(src.iter_data()) # fetch 2 — first combine - rows3 = list(src.iter_data()) # fetch 3 — second combine (crashed before fix) - - assert len(rows1) == 1 - assert len(rows2) == 2 - assert len(rows3) == 3 - - _, data_keys = src.keys() - assert "_content_hash" not in data_keys - - _, data_schema = src.output_schema() - assert "_content_hash" not in data_schema - - @pytest.mark.asyncio - async def test_async_three_fetches_no_content_hash_leak(self): - """After 3 async batches (2 combines), data schema is stable and contains - no _content_hash column, and all rows are accumulated. - - Regression test for ITL-616: same root cause as the sync path. - """ - fake = FakeDynamicSource( - batches=[_batch(1, 10), _batch(2, 20), _batch(3, 30)] - ) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig( - interval=0.05, duration=0.5, max_missed_intervals=50 - ), - ) - - items = [] - async for tag, data in src.async_iter_data(): - items.append((tag, data)) - - assert len(items) == 3 - assert src._accumulated_stream is not None - - _, data_keys = src._accumulated_stream.keys() - assert "_content_hash" not in data_keys - - _, data_schema = src._accumulated_stream.output_schema() - assert "_content_hash" not in data_schema - ``` - -- [ ] **Step 2: Run the tests to confirm they fail** - - ```bash - uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSchemaValidation::test_sync_three_fetches_no_content_hash_leak tests/test_channels/test_polling_source.py::TestPollingSourceSchemaValidation::test_async_three_fetches_no_content_hash_leak -v - ``` - - Expected: both tests **FAIL**. The sync test fails with `SchemaInconsistencyError` on the third `iter_data()` call. The async test fails similarly. - -- [ ] **Step 3: Commit the failing tests** - - ```bash - git add tests/test_channels/test_polling_source.py - git commit -m "test(polling_source): add three-fetch regression tests for content_hash leak (ITL-616)" - ``` - ---- - -### Task 3: Implement the fix and close out - -**Files:** -- Modify: `src/orcapod/core/sources/polling_source.py` -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Add `_STREAM_COMBINE_COLUMNS` constant to `polling_source.py`** - - Open `src/orcapod/core/sources/polling_source.py`. The file has three module-level - functions before the class: `_get_sync_executor`, `_run_sync`, and `_assert_schema_match`. - Find the end of `_assert_schema_match` (it ends with a `raise SchemaInconsistencyError(...)` - block) and the `# PollingSource` section comment that follows it: - - ```python - # --------------------------------------------------------------------------- - # PollingSource - # --------------------------------------------------------------------------- - - - class PollingSource(RootSource, Generic[T]): - ``` - - Insert the constant in the gap between `_assert_schema_match` and that section comment: - - ```python - # ColumnConfig used when concatenating streams in _combine. - # Includes the provenance columns (system_tags, source, context) that - # ArrowTableStream.__init__ knows how to parse and split into their - # respective internal tables. - # content_hash is intentionally absent: it is a synthetic, on-demand - # column produced by as_table(); including it would bake it into stored - # data and corrupt the data schema on the next combine. - _STREAM_COMBINE_COLUMNS = ColumnConfig(system_tags=True, source=True, context=True) - - - # --------------------------------------------------------------------------- - # PollingSource - # --------------------------------------------------------------------------- - ``` - - Note: `ColumnConfig` is already imported at the top of the file via - `from orcapod.types import ColumnConfig, Cursor, PollingConfig, Schema` — no new import needed. - -- [ ] **Step 2: Fix `_combine` to use `_STREAM_COMBINE_COLUMNS`** - - In the same file, locate the `_combine` method. It contains: - - ```python - combined = pa.concat_tables( - [ - existing.as_table(all_info=True), - new_stream.as_table(all_info=True), - ], - promote_options="default", - ) - ``` - - Replace with: - - ```python - combined = pa.concat_tables( - [ - existing.as_table(columns=_STREAM_COMBINE_COLUMNS), - new_stream.as_table(columns=_STREAM_COMBINE_COLUMNS), - ], - promote_options="default", - ) - ``` - - Nothing else in `_combine` changes. - -- [ ] **Step 3: Run the regression tests to confirm they now pass** - - ```bash - uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSchemaValidation::test_sync_three_fetches_no_content_hash_leak tests/test_channels/test_polling_source.py::TestPollingSourceSchemaValidation::test_async_three_fetches_no_content_hash_leak -v - ``` - - Expected: both tests **PASS**. - -- [ ] **Step 4: Run the full polling source test suite** - - ```bash - uv run pytest tests/test_channels/test_polling_source.py tests/test_channels/test_polling_source_pipeline_integration.py -v - ``` - - Expected: all tests **PASS**. No regressions. - -- [ ] **Step 5: Update PS3 status to `resolved` in `DESIGN_ISSUES.md`** - - Locate the PS3 entry added in Task 1. Change: - - ```markdown - **Status:** in progress - ``` - - to: - - ```markdown - **Status:** resolved - ``` - - Add a **Fix:** note immediately after the status line: - - ```markdown - **Status:** resolved - **Fix:** Added `_STREAM_COMBINE_COLUMNS = ColumnConfig(system_tags=True, source=True, - context=True)` constant and replaced `as_table(all_info=True)` with - `as_table(columns=_STREAM_COMBINE_COLUMNS)` in `_combine`. `content_hash` is - intentionally excluded — it is a synthetic output column, never a stored one. - ``` - -- [ ] **Step 6: Commit the fix** - - ```bash - git add src/orcapod/core/sources/polling_source.py DESIGN_ISSUES.md - git commit -m "fix(polling_source): exclude _content_hash from _combine column config (ITL-616) - - _combine passed all_info=True to as_table(), which triggered content_hash=True - and injected the synthetic _content_hash column into the concatenated table. - ArrowTableStream.__init__ stored it as a data column, corrupting the schema - on the second combine and raising SchemaInconsistencyError on the third fetch. - - Add _STREAM_COMBINE_COLUMNS = ColumnConfig(system_tags=True, source=True, - context=True) and use it in place of all_info=True. content_hash is excluded - because it is a synthetic, on-demand output — not a stored column." - ``` - -- [ ] **Step 7: Run the full test suite** - - ```bash - uv run pytest tests/ -x -q - ``` - - Expected: all tests pass. diff --git a/superpowers/plans/2026-08-25-itl-617-pollingsource-concurrent-cursor-safety.md b/superpowers/plans/2026-08-25-itl-617-pollingsource-concurrent-cursor-safety.md deleted file mode 100644 index 762f71bc8..000000000 --- a/superpowers/plans/2026-08-25-itl-617-pollingsource-concurrent-cursor-safety.md +++ /dev/null @@ -1,834 +0,0 @@ -# ITL-617: PollingSource concurrent cursor safety — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace `PollingSource`'s single shared `_accumulated_stream` cache with an append-only `_batches` list and an optimistic lock on `_cursor`, so concurrent sync callers cannot silently steal rows from the async polling loop. - -**Architecture:** All mutation of `_cursor` and `_batches` is protected by `_state_lock: threading.Lock` using the optimistic lock protocol: snapshot cursor (brief lock) → I/O with no lock held → commit only if cursor unchanged (brief lock). The async loop tracks its yield position with a per-iterator `local_batch_idx` local variable and drains `_batches` at the top of each iteration, so rows committed by a concurrent sync caller are still yielded even if the async loop lost the commit race. - -**Tech Stack:** Python `threading.Lock`, `asyncio`, PyArrow, pytest-asyncio. - ---- - -## Files - -| File | What changes | -|------|-------------| -| `src/orcapod/core/sources/polling_source.py` | Replace `_accumulated_stream` with `_batches` + `_state_lock`; add `_sync_poll_and_commit()`, `_get_combined_stream()`; rewrite `async_iter_data()`; update `iter_data()`, `as_table()`, `output_schema()`, `keys()` | -| `tests/test_channels/test_polling_source.py` | Fix `test_cache_combining_accumulates_rows` (accesses `_accumulated_stream` directly); add `TestPollingSourceSyncAccessDuringAsyncRun` | -| `DESIGN_ISSUES.md` | Add PS4 entry | - ---- - -## Task 1: Create feature branch - -**Files:** — (git only) - -- [ ] **Step 1: Checkout the feature branch** - -```bash -cd /home/kurouto/kurouto-jobs/15fa2ef8-6c81-4b68-81f3-cc9492a3dc2b/orcapod-python -git checkout eywalker/itl-617-pollingsource-sync-introspection-during-an-async-run -``` - -Expected: branch switches without error (branch already exists from ITL-617 issue). -If the branch doesn't exist yet, create it from main: -```bash -git checkout main && git pull && git checkout -b eywalker/itl-617-pollingsource-sync-introspection-during-an-async-run -``` - -- [ ] **Step 2: Confirm baseline test suite passes** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v -``` - -Expected: all tests **PASS**. Record the count — we must not regress any. - ---- - -## Task 2: Write failing concurrent-access tests - -Write the new test class first so we have a red light that turns green after the fix. These tests expose the current bug: a sync caller can steal a row that the async loop should emit. - -**Files:** -- Modify: `tests/test_channels/test_polling_source.py` (add class at end of file) - -- [ ] **Step 1: Append `TestPollingSourceSyncAccessDuringAsyncRun` to the test file** - -Add the following class at the very end of `tests/test_channels/test_polling_source.py`: - -```python -# =========================================================================== -# ITL-617: Concurrent sync access during async run must not lose rows -# =========================================================================== - - -class TestPollingSourceSyncAccessDuringAsyncRun: - """Regression tests for ITL-617. - - A concurrent sync call (``iter_data``, ``as_table``, ``output_schema``, - ``keys``) must not advance ``_cursor`` in a way that causes the async - polling loop to skip rows. - """ - - @pytest.mark.asyncio - async def test_iter_data_and_as_table_concurrent_with_async_run_lose_no_rows(self): - """iter_data() and as_table() called concurrently with async_iter_data() - must not cause any rows to be skipped by the async iterator.""" - fake = FakeDynamicSource( - batches=[_batch(1, 10), _batch(2, 20), _batch(3, 30)], - schema_override=None, - ) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.02, duration=1.0, max_missed_intervals=100), - ) - - rows_from_async: list = [] - stop_bg = asyncio.Event() - - async def background_sync_calls(): - # Wait until at least one batch is committed, then hammer the - # sync API from a thread pool (iter_data and as_table both call - # _sync_poll_and_commit which can race with the async loop). - while not src._batches: - await asyncio.sleep(0.005) - while not stop_bg.is_set(): - await asyncio.to_thread(lambda: list(src.iter_data())) - await asyncio.to_thread(lambda: src.as_table()) - await asyncio.sleep(0.005) - - bg_task = asyncio.create_task(background_sync_calls()) - - async for tag, data in src.async_iter_data(): - rows_from_async.append((tag, data)) - - stop_bg.set() - try: - await asyncio.wait_for(bg_task, timeout=1.0) - except asyncio.TimeoutError: - bg_task.cancel() - - # The async iterator must deliver ALL three rows, regardless of - # how many times the sync caller raced in. - assert len(rows_from_async) == 3 - - @pytest.mark.asyncio - async def test_output_schema_and_keys_concurrent_with_async_run_lose_no_rows(self): - """output_schema() and keys() called concurrently with async_iter_data() - must not trigger a fetch that advances the cursor past the async loop.""" - fake = FakeDynamicSource( - batches=[_batch(1, 10), _batch(2, 20), _batch(3, 30)], - schema_override=None, # no declared schema → would fall through to fetch - ) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.02, duration=1.0, max_missed_intervals=100), - ) - - rows_from_async: list = [] - stop_bg = asyncio.Event() - - async def background_introspection(): - # Wait until first batch is available (so _batches[0] exists) - # then hammer output_schema / keys. - while not src._batches: - await asyncio.sleep(0.005) - while not stop_bg.is_set(): - src.output_schema() - src.keys() - src.output_schema(columns={"system_tags": True}) - src.keys(columns={"system_tags": True}) - await asyncio.sleep(0.005) - - bg_task = asyncio.create_task(background_introspection()) - - async for tag, data in src.async_iter_data(): - rows_from_async.append((tag, data)) - - stop_bg.set() - try: - await asyncio.wait_for(bg_task, timeout=1.0) - except asyncio.TimeoutError: - bg_task.cancel() - - # All 3 rows must be delivered by the async iterator. - assert len(rows_from_async) == 3 - # output_schema / keys must not have triggered any fetches — - # they should use _batches[0] bypass once the first batch exists. - # Exactly 3 fetches: one per batch, all by the async loop. - assert len(fake.fetch_cursors) == 3 -``` - -- [ ] **Step 2: Run the new tests to confirm they fail (exposing the bug)** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSyncAccessDuringAsyncRun -v -``` - -Expected: both tests **FAIL**. -- `test_iter_data_and_as_table_concurrent_with_async_run_lose_no_rows`: `assert len(rows_from_async) == 3` fails (rows_from_async has 2 items — sync caller stole one batch). -- `test_output_schema_and_keys_concurrent_with_async_run_lose_no_rows`: may fail depending on timing, but `len(fake.fetch_cursors) == 3` may be > 3. - -If both tests happen to pass (race didn't trigger) — that's also acceptable; the implementation still needs the fix for correctness. - ---- - -## Task 3: Replace `_accumulated_stream` with `_batches` + `_state_lock` in `__init__` - -**Files:** -- Modify: `src/orcapod/core/sources/polling_source.py` - -- [ ] **Step 1: Add `import threading` to the module-level imports** - -In `polling_source.py`, find the imports block (lines 9–26) and add `threading`: - -```python -import asyncio -import dataclasses -import functools -import logging -import threading -from collections.abc import Collection -from math import floor -from typing import TYPE_CHECKING, Any, Generic, TypeVar -``` - -- [ ] **Step 2: Replace `_accumulated_stream` with `_batches` and `_state_lock` in `__init__`** - -In `PollingSource.__init__`, find: -```python - self._cursor: Cursor[T] | None = None - self._accumulated_stream: ArrowTableStream | None = None -``` - -Replace with: -```python - self._cursor: Cursor[T] | None = None - self._batches: list[ArrowTableStream] = [] - self._state_lock: threading.Lock = threading.Lock() -``` - -- [ ] **Step 3: Update the cached-stream bypass in `output_schema()`** - -In `output_schema()`, find (around line 383): -```python - if self._accumulated_stream is not None: - return self._accumulated_stream.output_schema(columns=columns, all_info=all_info) - return self._get_latest_stream().output_schema(columns=columns, all_info=all_info) -``` - -Replace with: -```python - if self._batches: - return self._batches[0].output_schema(columns=columns, all_info=all_info) - return self._get_latest_stream().output_schema(columns=columns, all_info=all_info) -``` - -- [ ] **Step 4: Update the cached-stream bypass in `keys()`** - -In `keys()`, find (around line 417): -```python - if self._accumulated_stream is not None: - return self._accumulated_stream.keys(columns=columns, all_info=all_info) - return self._get_latest_stream().keys(columns=columns, all_info=all_info) -``` - -Replace with: -```python - if self._batches: - return self._batches[0].keys(columns=columns, all_info=all_info) - return self._get_latest_stream().keys(columns=columns, all_info=all_info) -``` - -- [ ] **Step 5: Fix the existing test that directly accesses `_accumulated_stream`** - -In `tests/test_channels/test_polling_source.py`, find `test_cache_combining_accumulates_rows` (around line 583). It ends with: - -```python - assert src._accumulated_stream is not None - cached_rows = list(src._accumulated_stream.iter_data()) - assert len(cached_rows) == 2 -``` - -Replace those three lines with: - -```python - assert len(src._batches) == 2 - cached_rows = list(src._get_combined_stream().iter_data()) - assert len(cached_rows) == 2 -``` - -(Note: `_get_combined_stream()` is added in Task 4 — for now this test will still fail. That's expected.) - -- [ ] **Step 6: Run the sync test suite to check compile-time errors only** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v --tb=short 2>&1 | head -60 -``` - -Expected: tests fail at runtime (AttributeError on `_accumulated_stream` and `_get_combined_stream` not yet defined), but no import errors. This is expected — we'll fix in the next tasks. - ---- - -## Task 4: Add `_sync_poll_and_commit()` and `_get_combined_stream()` - -These are the two new internal helpers that replace `_get_latest_stream()` for the mutation and read paths respectively. - -**Files:** -- Modify: `src/orcapod/core/sources/polling_source.py` - -- [ ] **Step 1: Add `_sync_poll_and_commit()` method** - -Insert this method in the "Internal sync helpers" section, immediately before `_get_latest_stream()`: - -```python - def _sync_poll_and_commit(self) -> None: - """Poll for new data and commit a new batch under the optimistic lock. - - Implements the check-snapshot → I/O → check-and-commit protocol: - reads the cursor snapshot under a brief lock, performs poll and fetch - with no lock held, then commits only if the cursor has not changed. - If another caller advanced the cursor in between (winning the commit - race), this call discards its fetched data — the batch is already in - ``_batches`` and will be visible on the next read. - - Safe to call concurrently with ``async_iter_data``. - """ - with self._state_lock: - cursor_snapshot = self._cursor - - if cursor_snapshot is None: - # First access — no poll needed, fetch unconditionally. - logger.debug("PollingSource %r: first sync access — fetching", self._source_id) - new_cursor, data = _run_sync(self._impl.fetch, cursor=None) - new_stream = self._try_build_stream(data) - if new_stream is not None: - if self._tag_schema is not None or self._data_schema is not None: - self._validate_against_declared_schemas(new_stream) - committed = False - with self._state_lock: - if self._cursor is None: - if new_stream is not None: - self._batches.append(new_stream) - self._cursor = new_cursor - committed = True - if committed: - self._update_last_modified_from_cursor(new_cursor) - else: - has_new = _run_sync(self._impl.poll, cursor=cursor_snapshot) - if has_new: - logger.debug( - "PollingSource %r: sync poll found new data — fetching", self._source_id - ) - new_cursor, data = _run_sync(self._impl.fetch, cursor=cursor_snapshot) - new_stream = self._try_build_stream(data) - if new_stream is not None: - if self._tag_schema is not None or self._data_schema is not None: - self._validate_against_declared_schemas(new_stream) - if self._batches: - self._validate_combining_schemas(self._batches[0], new_stream) - committed = False - with self._state_lock: - if self._cursor == cursor_snapshot: - if new_stream is not None: - self._batches.append(new_stream) - self._cursor = new_cursor - committed = True - if committed: - self._update_last_modified_from_cursor(new_cursor) - else: - logger.debug( - "PollingSource %r: sync poll — cache still valid", self._source_id - ) -``` - -- [ ] **Step 2: Add `_get_combined_stream()` method** - -Insert this method immediately after `_sync_poll_and_commit()`: - -```python - def _get_combined_stream(self) -> ArrowTableStream: - """Return all committed batches concatenated as a single stream. - - Takes a snapshot of ``_batches`` (no lock needed — list is append-only) - then combines using the existing ``_combine`` helper. - - Returns: - A single ``ArrowTableStream`` containing all rows from all batches. - - Raises: - ValueError: If no data has been fetched yet (``_batches`` is empty). - """ - batches = list(self._batches) # snapshot — safe, list is append-only - if not batches: - raise ValueError( - "PollingSource: no data available yet — first fetch returned empty data." - ) - result = batches[0] - for batch in batches[1:]: - result = self._combine(result, batch) - return result -``` - -- [ ] **Step 3: Run existing sync tests to verify basic behavior is intact** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSyncMode -v -``` - -These tests still call `_get_latest_stream()` via `iter_data()` / `as_table()` (not yet updated). Expected: they may still fail because `iter_data()` / `as_table()` still use the old path. We will fix this in Task 5. - ---- - -## Task 5: Update `iter_data()`, `as_table()`, and remove `_get_latest_stream()` - -**Files:** -- Modify: `src/orcapod/core/sources/polling_source.py` - -- [ ] **Step 1: Update `iter_data()` to use the new helpers** - -Find: -```python - def iter_data(self): - """Iterate over (tag, data) pairs from the current snapshot.""" - return self._get_latest_stream().iter_data() -``` - -Replace with: -```python - def iter_data(self): - """Iterate over (tag, data) pairs from the current snapshot.""" - self._sync_poll_and_commit() - return self._get_combined_stream().iter_data() -``` - -- [ ] **Step 2: Update `as_table()` to use the new helpers** - -Find: -```python - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table: - """Return the accumulated rows as a PyArrow table.""" - return self._get_latest_stream().as_table(columns=columns, all_info=all_info) -``` - -Replace with: -```python - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table: - """Return the accumulated rows as a PyArrow table.""" - self._sync_poll_and_commit() - return self._get_combined_stream().as_table(columns=columns, all_info=all_info) -``` - -- [ ] **Step 3: Delete `_get_latest_stream()`** - -Remove the entire `_get_latest_stream` method (lines ~513–546 in the original). It is no longer called anywhere. - -- [ ] **Step 4: Run sync-mode tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSyncMode -v -``` - -Expected: all **PASS**. - -- [ ] **Step 5: Run schema validation tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceSchemaValidation -v -``` - -Expected: all **PASS** (including `test_sync_three_fetches_no_content_hash_leak`). - ---- - -## Task 6: Rewrite `async_iter_data()` - -Replace the pre-seed block and the poll/fetch/commit section with the optimistic lock protocol and per-iterator `local_batch_idx`. All error handling (cancelled, cursor invalidated, schema mismatch, exponential backoff, overrun, duration) is preserved verbatim. - -**Files:** -- Modify: `src/orcapod/core/sources/polling_source.py` - -- [ ] **Step 1: Replace the body of `async_iter_data()`** - -Find the entire `async_iter_data` method (lines ~683–849) and replace it with: - -```python - async def async_iter_data(self): - """Async generator that continuously emits (tag, data) pairs. - - Uses a per-iterator ``local_batch_idx`` to track the next position in - the append-only ``_batches`` list. A drain step at the top of each - loop iteration yields any newly committed batches (including those - committed by concurrent sync callers) before sleeping. - - The optimistic lock protocol — snapshot cursor (brief lock) → poll and - fetch with no lock held → commit only if cursor unchanged (brief lock) - — prevents TOCTOU races without ever holding ``_state_lock`` across an - ``await``. - - ``impl.close()`` is always awaited before returning or raising. - """ - # local_batch_idx tracks the next _batches index to yield. - # Starting at 0 means the first drain covers any pre-existing batches - # (the pre-seed case from the old implementation). - local_batch_idx = 0 - - cfg = self._polling_config - loop = asyncio.get_running_loop() - start_time = loop.time() - next_tick = start_time - consecutive_misses = 0 - consecutive_errors = 0 - - logger.info( - "PollingSource %r starting (interval=%.2fs, duration=%.1fs)", - self._source_id, - cfg.interval, - cfg.duration, - ) - - try: - while True: - # ── 1. Drain: yield any batches not yet emitted ────────────── - # Covers pre-existing rows on first iteration AND rows committed - # by concurrent sync callers while this iterator was polling. - while local_batch_idx < len(self._batches): - for item in self._batches[local_batch_idx].iter_data(): - yield item - local_batch_idx += 1 - - # ── 2. Sleep to next scheduled tick ────────────────────────── - now = loop.time() - if next_tick > now: - await asyncio.sleep(next_tick - now) - - # ── 3. Poll + fetch + commit (optimistic lock) ─────────────── - try: - # Brief lock: snapshot cursor only. - with self._state_lock: - cursor_snapshot = self._cursor - - # Poll — no lock held across await. - has_new = await self._impl.poll(cursor=cursor_snapshot) - - if has_new: - logger.debug( - "PollingSource %r: new data detected, fetching", - self._source_id, - ) - # Fetch — no lock held across await. - new_cursor, data = await self._impl.fetch(cursor=cursor_snapshot) - new_stream = self._try_build_stream(data) - - if new_stream is not None: - if self._tag_schema is not None or self._data_schema is not None: - self._validate_against_declared_schemas(new_stream) - if self._batches: - self._validate_combining_schemas( - self._batches[0], new_stream - ) - - # Brief lock: check cursor then commit atomically. - committed = False - with self._state_lock: - if self._cursor == cursor_snapshot: - if new_stream is not None: - self._batches.append(new_stream) - self._cursor = new_cursor - committed = True - # else: sync caller already advanced cursor — - # their batch is in _batches; drain step above - # will yield it at the top of the next iteration. - - if committed: - self._update_last_modified_from_cursor(new_cursor) - emitted_count = new_stream.as_table().num_rows if new_stream is not None else 0 - logger.debug( - "PollingSource %r: committed %d row(s)", - self._source_id, - emitted_count, - ) - else: - logger.debug( - "PollingSource %r: poll returned no new data", - self._source_id, - ) - - consecutive_errors = 0 - - except asyncio.CancelledError: - raise - - except CursorInvalidatedError: - logger.error( - "PollingSource %r: cursor invalidated — previous state cannot " - "be reconciled with already-emitted rows. Terminating source.", - self._source_id, - ) - raise - - except InputValidationError: - # Schema mismatches are not transient — propagate immediately. - raise - - except Exception as e: - consecutive_errors += 1 - backoff = cfg.error_backoff_base * 2 ** (consecutive_errors - 1) - logger.error( - "PollingSource %r: poll/fetch error (consecutive=%d, " - "backoff=%.1fs): %s", - self._source_id, - consecutive_errors, - backoff, - e, - ) - if consecutive_errors >= cfg.max_consecutive_errors: - logger.error( - "PollingSource %r: max consecutive errors (%d) reached. " - "Terminating source.", - self._source_id, - cfg.max_consecutive_errors, - ) - return - await asyncio.sleep(backoff) - continue # retry — do not advance next_tick - - # ── 4. Tick advancement (start-to-start) ───────────────────── - now = loop.time() - intervals_consumed = floor((now - next_tick) / cfg.interval) - if intervals_consumed > 0: - consecutive_misses += intervals_consumed - logger.warning( - "PollingSource %r: tick overrun — consumed %d interval(s) " - "(consecutive_misses=%d/%d)", - self._source_id, - intervals_consumed, - consecutive_misses, - cfg.max_missed_intervals, - ) - if consecutive_misses >= cfg.max_missed_intervals: - logger.error( - "PollingSource %r: overrun threshold exceeded. " - "Terminating source.", - self._source_id, - ) - return - else: - consecutive_misses = 0 - next_tick += (intervals_consumed + 1) * cfg.interval - - # ── 5. Duration check ───────────────────────────────────────── - if cfg.duration > 0 and (loop.time() - start_time) >= cfg.duration: - logger.info( - "PollingSource %r: duration limit (%.1fs) reached. " - "Terminating source.", - self._source_id, - cfg.duration, - ) - return - - except asyncio.CancelledError: - logger.info( - "PollingSource %r: cancelled — shutting down cleanly.", - self._source_id, - ) - - finally: - logger.debug("PollingSource %r: calling impl.close()", self._source_id) - await self._impl.close() - logger.info("PollingSource %r: closed.", self._source_id) -``` - -- [ ] **Step 2: Run all async mode tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceAsyncMode -v -``` - -Expected: all **PASS**. - -Note: `test_pre_seeding_yields_cached_rows_first` seeds the cache with `list(src.iter_data())` then replaces `src._impl`. The drain step at `local_batch_idx=0` yields the pre-seeded row first, so this test still passes. - -Note: `test_cache_combining_accumulates_rows` now checks `src._batches` and `src._get_combined_stream()` — it should **PASS** after Task 3 step 5 update. - -- [ ] **Step 3: Run error handling tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceErrorHandling -v -``` - -Expected: all **PASS**. - ---- - -## Task 7: Run the full test suite and verify - -**Files:** — (verification only) - -- [ ] **Step 1: Run the entire polling source test file** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v -``` - -Expected: all tests **PASS**, including the two new `TestPollingSourceSyncAccessDuringAsyncRun` tests. - -- [ ] **Step 2: Run the broader test suite to check for regressions** - -```bash -uv run pytest tests/ -x -q --timeout=120 -``` - -Expected: all tests **PASS**. If any fail, investigate before proceeding. - ---- - -## Task 8: Update `DESIGN_ISSUES.md` - -**Files:** -- Modify: `DESIGN_ISSUES.md` - -- [ ] **Step 1: Add PS4 entry after PS3 in the `polling_source.py` section** - -In `DESIGN_ISSUES.md`, find the line: -``` ---- - -## `src/orcapod/core/nodes/function_node.py` -``` - -(immediately after the PS3 entry). Insert this new entry before that separator: - -```markdown -### PS4 — Concurrent sync access during async run silently loses rows -**Status:** resolved -**Severity:** critical -**Issue:** ITL-617 - -`_get_latest_stream()`, called by `iter_data()` or `as_table()` while -`async_iter_data()` is running, could advance `_cursor` and fold new rows into -`_accumulated_stream` without the async loop emitting them — permanent silent data -loss. `ITL-615` (PR #255) partially addressed this for `output_schema()` and `keys()` -by caching the stream reference. `ITL-617` fixes the root. - -**Fix:** Replaced single `_accumulated_stream: ArrowTableStream | None` with -append-only `_batches: list[ArrowTableStream]` and `_state_lock: threading.Lock`. -All callers (sync and async) use an optimistic lock protocol: snapshot cursor (brief -lock) → perform I/O freely with no lock held → commit only if cursor is unchanged -(brief lock). The async loop tracks its yield position with a per-iterator -``local_batch_idx`` local variable and drains ``_batches`` at the top of each -iteration, ensuring rows committed by a concurrent sync caller are yielded even when -the async loop loses the commit race. The lock is never held across ``await``. - ---- -``` - -- [ ] **Step 2: Verify the DESIGN_ISSUES.md renders cleanly** - -```bash -uv run python -c "open('DESIGN_ISSUES.md').read(); print('OK')" -``` - -Expected: `OK` (file is readable, no encoding issues). - ---- - -## Task 9: Commit - -**Files:** — (git only) - -- [ ] **Step 1: Check which files changed** - -```bash -git diff --stat -``` - -Expected: 3 files — `src/orcapod/core/sources/polling_source.py`, `tests/test_channels/test_polling_source.py`, `DESIGN_ISSUES.md`. - -- [ ] **Step 2: Run full tests one final time** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v -q -``` - -Expected: all **PASS**. - -- [ ] **Step 3: Stage and commit** - -```bash -git add src/orcapod/core/sources/polling_source.py \ - tests/test_channels/test_polling_source.py \ - DESIGN_ISSUES.md \ - superpowers/specs/2026-08-25-itl-617-pollingsource-async-mode-guard-design.md \ - superpowers/plans/2026-08-25-itl-617-pollingsource-concurrent-cursor-safety.md -git commit -m "$(cat <<'EOF' -fix(polling_source): replace _accumulated_stream with optimistic-lock batch list - -Fixes ITL-617: concurrent sync callers (iter_data, as_table) could advance -_cursor while async_iter_data was running, causing the async loop to skip -already-fetched rows — permanent silent data loss. - -Replace the single _accumulated_stream with an append-only _batches list and -a threading.Lock. All callers use the optimistic lock protocol: snapshot cursor -(brief lock) → I/O with no lock held → commit only if cursor unchanged (brief -lock). The async loop tracks its position with a per-iterator local_batch_idx -and drains _batches at the top of each iteration to pick up sync-committed -rows. The lock is never held across await. - -Closes ITL-617 -EOF -)" -``` - -- [ ] **Step 4: Verify commit looks right** - -```bash -git log --oneline -3 -git show --stat HEAD -``` - -Expected: one new commit with the 5 files listed. - ---- - -## Self-Review - -**Spec coverage checklist:** - -| Spec requirement | Task | -|---|---| -| `_accumulated_stream` → `_batches` (append-only) | Task 3 step 2 | -| `_state_lock: threading.Lock` added | Task 3 step 2 | -| `output_schema()` cached bypass → `_batches[0]` | Task 3 step 3 | -| `keys()` cached bypass → `_batches[0]` | Task 3 step 4 | -| `_sync_poll_and_commit()` — optimistic lock protocol | Task 4 step 1 | -| `_get_combined_stream()` — concat all batches | Task 4 step 2 | -| `iter_data()` uses `_sync_poll_and_commit` + `_get_combined_stream` | Task 5 step 1 | -| `as_table()` uses `_sync_poll_and_commit` + `_get_combined_stream` | Task 5 step 2 | -| `_get_latest_stream()` removed | Task 5 step 3 | -| `async_iter_data()` rewritten: `local_batch_idx`, drain step, opt-lock | Task 6 step 1 | -| Lock never held across `await` | Task 6 step 1 | -| No `_async_loop_active` flag | (design choice — not in plan) | -| `test_cache_combining_accumulates_rows` updated | Task 3 step 5 | -| `TestPollingSourceSyncAccessDuringAsyncRun` with 2 tests | Task 2 step 1 | -| `DESIGN_ISSUES.md` PS4 entry | Task 8 step 1 | - -All spec requirements are covered. - -**Type consistency check:** `_batches` is typed as `list[ArrowTableStream]`. `_batches[0]` returns `ArrowTableStream`. `_get_combined_stream()` returns `ArrowTableStream`. `_combine(result, batch)` takes two `ArrowTableStream` and returns `ArrowTableStream`. All consistent. ✓ - -**`_validate_combining_schemas(self._batches[0], new_stream)`** — called in `_sync_poll_and_commit()` and `async_iter_data()` before the commit lock. The method signature is `_validate_combining_schemas(existing: ArrowTableStream, new_stream: ArrowTableStream)`. ✓ diff --git a/superpowers/plans/2026-08-27-eng-952-polling-source-zero-row-batch-fix.md b/superpowers/plans/2026-08-27-eng-952-polling-source-zero-row-batch-fix.md deleted file mode 100644 index cdfcff658..000000000 --- a/superpowers/plans/2026-08-27-eng-952-polling-source-zero-row-batch-fix.md +++ /dev/null @@ -1,528 +0,0 @@ -# ENG-952: PollingSource Zero-Row Batch Fix Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Fix `PollingSource` crashing with `SchemaInconsistencyError` on zero-row polls by establishing a canonical Arrow schema once and applying it to every batch. - -**Architecture:** Add `_canonical_arrow_schema: pa.Schema | None` to `PollingSource.__init__`. Modify `_build_stream_from_df` to set the canonical schema on the first call (from `impl.schema()` when declared, or inferred from the first batch with a `WARNING`) and cast every subsequent batch to it by column name. No other methods change. - -**Tech Stack:** PyArrow (`pa.Schema`, `Table.cast`), Polars (`pl.DataFrame`), pytest-asyncio, pytest `caplog` fixture. - ---- - -## Files - -| File | Change | -|------|--------| -| `src/orcapod/core/sources/polling_source.py` | Add `_canonical_arrow_schema` attribute; rewrite the single `infer_schema_nullable` call in `_build_stream_from_df` | -| `tests/test_channels/test_polling_source.py` | Add `TestPollingSourceZeroRowBatch` class with four tests | -| `DESIGN_ISSUES.md` | Add PS4 entry (resolved) | - ---- - -### Task 1: Write the four failing tests - -**Files:** -- Modify: `tests/test_channels/test_polling_source.py` (append after line 1144) - -- [ ] **Step 1: Append `TestPollingSourceZeroRowBatch` to the test file** - -Add the following class at the very end of `tests/test_channels/test_polling_source.py` (after the `_drain_async` helper): - -```python -# =========================================================================== -# Task N: Zero-row batch regression (ENG-952) -# =========================================================================== - - -class TestPollingSourceZeroRowBatch: - """Regression tests for ENG-952. - - PollingSource must not crash when a poll returns a zero-row batch after - a batch that contained a null value in some column. The root cause was - per-batch nullability re-inference: a zero-row table has null_count == 0 - for every column, so every field was inferred non-nullable, conflicting - with the accumulated stream's nullable schema. - - The fix establishes a canonical Arrow schema once (from impl.schema() when - declared, or from the first batch otherwise) and casts every subsequent - batch to it by column name. - """ - - # ------------------------------------------------------------------ - # Shared impl used by tests 1 and 2 (infer-once path, no declared schema) - # ------------------------------------------------------------------ - - @staticmethod - def _make_emit_once_then_empty_impl(): - """Return a DynamicSourceProtocol impl that emits one nullable row then empty frames.""" - - class EmitOnceThenEmpty: - """Emits one row with a null 'note' on fetch 1, then zero-row frames.""" - - def __init__(self): - self.n = 0 - - def identity(self): - return ("EmitOnceThenEmpty",) - - def to_config(self): - return None - - @classmethod - def from_config(cls, config): - raise NotImplementedError - - def schema(self): - return None # no declared schema — exercises the infer-once path - - async def poll(self, cursor=None): - return True # always claims new data - - async def fetch(self, cursor=None): - self.n += 1 - if self.n == 1: - # First fetch: one row, nullable 'note' column contains None. - data = { - "id": pa.array([1], type=pa.int64()), - "val": pa.array([1.0], type=pa.float64()), - "note": pa.array([None], type=pa.large_utf8()), - } - else: - # Subsequent fetches: zero-row frame, same columns. - data = { - "id": pa.array([], type=pa.int64()), - "val": pa.array([], type=pa.float64()), - "note": pa.array([], type=pa.large_utf8()), - } - return Cursor.now(self.n), data - - async def close(self): - return None - - return EmitOnceThenEmpty() - - # ------------------------------------------------------------------ - # Test 1: core regression — source must not crash - # ------------------------------------------------------------------ - - @pytest.mark.asyncio - async def test_zero_row_batch_after_nullable_column_streams_cleanly(self): - """Zero-row poll after a nullable column must not raise SchemaInconsistencyError. - - This is the exact scenario from ENG-952: fetch 1 returns a row with a - null in 'note' (inferred nullable=True); fetches 2+ return zero-row frames - (would be inferred nullable=False without the fix). The source must stream - the single real row and then terminate cleanly after the duration expires. - """ - src = PollingSource( - self._make_emit_once_then_empty_impl(), - tag_columns="id", - polling_config=PollingConfig(interval=0.05, duration=0.5, max_missed_intervals=50), - ) - - rows = [] - async for tag, data in src.async_iter_data(): - rows.append((tag, data)) - - assert len(rows) == 1 - - # ------------------------------------------------------------------ - # Test 2: zero-row batches must not accumulate - # ------------------------------------------------------------------ - - @pytest.mark.asyncio - async def test_zero_row_batch_is_not_accumulated(self): - """After zero-row polls, the internal accumulated stream must hold only the real rows.""" - src = PollingSource( - self._make_emit_once_then_empty_impl(), - tag_columns="id", - polling_config=PollingConfig(interval=0.05, duration=0.5, max_missed_intervals=50), - ) - - async for _ in src.async_iter_data(): - pass - - assert src._accumulated_stream is not None - cached = list(src._accumulated_stream.iter_data()) - assert len(cached) == 1 - - # ------------------------------------------------------------------ - # Test 3: declared-schema path — zero-row polls, no warning - # ------------------------------------------------------------------ - - @pytest.mark.asyncio - async def test_declared_schema_zero_row_batch_no_warning(self, caplog): - """Declared-schema path: zero-row polls stream cleanly and emit no WARNING. - - When impl.schema() returns a Schema, nullability is derived from the - Python type annotations (str | None → nullable=True) without inference. - No warning must be logged even when the first batch carries a null. - """ - - class DeclaredNullableImpl: - def __init__(self): - self.n = 0 - - def identity(self): - return ("DeclaredNullableImpl",) - - def to_config(self): - return None - - @classmethod - def from_config(cls, config): - raise NotImplementedError - - def schema(self): - # note is declared nullable via str | None - return Schema({"id": int, "val": float, "note": str | None}) - - async def poll(self, cursor=None): - return True - - async def fetch(self, cursor=None): - self.n += 1 - if self.n == 1: - data = { - "id": pa.array([1], type=pa.int64()), - "val": pa.array([1.0], type=pa.float64()), - "note": pa.array([None], type=pa.large_utf8()), - } - else: - data = { - "id": pa.array([], type=pa.int64()), - "val": pa.array([], type=pa.float64()), - "note": pa.array([], type=pa.large_utf8()), - } - return Cursor.now(self.n), data - - async def close(self): - return None - - src = PollingSource( - DeclaredNullableImpl(), - tag_columns="id", - polling_config=PollingConfig(interval=0.05, duration=0.5, max_missed_intervals=50), - ) - - with caplog.at_level(logging.WARNING, logger="orcapod.core.sources.polling_source"): - rows = [] - async for tag, data in src.async_iter_data(): - rows.append((tag, data)) - - assert len(rows) == 1 - inference_warnings = [ - r for r in caplog.records if "inferring nullability" in r.message - ] - assert len(inference_warnings) == 0, ( - "Declared-schema path must not emit an inference warning" - ) - - # ------------------------------------------------------------------ - # Test 4: infer-once path emits exactly one WARNING - # ------------------------------------------------------------------ - - @pytest.mark.asyncio - async def test_infer_once_emits_exactly_one_warning(self, caplog): - """When impl.schema() returns None, exactly one WARNING is logged for schema inference. - - The warning must be emitted on the first batch only — not on every subsequent - poll — because _canonical_arrow_schema is set after the first call. - """ - # Two batches so _build_stream_from_df is called twice; warning fires only once. - fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) - src = PollingSource( - fake, - tag_columns="id", - polling_config=PollingConfig(interval=0.05, duration=0.5, max_missed_intervals=50), - ) - - with caplog.at_level(logging.WARNING, logger="orcapod.core.sources.polling_source"): - async for _ in src.async_iter_data(): - pass - - inference_warnings = [ - r for r in caplog.records if "inferring nullability" in r.message - ] - assert len(inference_warnings) == 1, ( - f"Expected exactly one inference warning, got {len(inference_warnings)}" - ) -``` - -- [ ] **Step 2: Run all four new tests to confirm they fail** - -```bash -cd /path/to/orcapod-python -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceZeroRowBatch -v -``` - -Expected: all four tests **FAIL**. - -- Tests 1–3 should fail with `SchemaInconsistencyError` (or the underlying error propagation). -- Test 4 should fail with `AssertionError: Expected exactly one inference warning, got 0`. - -If any test passes before the fix is applied, stop and investigate — the test may not be covering the right behaviour. - -- [ ] **Step 3: Commit the failing tests** - -```bash -git add tests/test_channels/test_polling_source.py -git commit -m "test(polling_source): add failing regression tests for ENG-952 zero-row batch crash" -``` - ---- - -### Task 2: Implement the canonical-schema fix in `PollingSource` - -**Files:** -- Modify: `src/orcapod/core/sources/polling_source.py` - -- [ ] **Step 1: Add `_canonical_arrow_schema` attribute to `__init__`** - -In `PollingSource.__init__`, add one line immediately after the `_accumulated_stream` assignment (line 258): - -```python - self._accumulated_stream: ArrowTableStream | None = None - self._canonical_arrow_schema: pa.Schema | None = None # add this line -``` - -The full surrounding context (lines 255–262) should look like: - -```python - self._impl: DynamicSourceProtocol[T] = impl - self._tag_columns: tuple[str, ...] = tuple(_normalize_column_list(tag_columns)) - self._polling_config = polling_config - self._cursor: Cursor[T] | None = None - self._accumulated_stream: ArrowTableStream | None = None - self._canonical_arrow_schema: pa.Schema | None = None - # Derive source_id from impl identity if not explicitly provided - if self._source_id is None: - self._source_id = str(self._impl.identity()) -``` - -- [ ] **Step 2: Replace the single `infer_schema_nullable` call in `_build_stream_from_df`** - -Locate line 578 in `polling_source.py`: - -```python - arrow_table = arrow_table.cast(arrow_utils.infer_schema_nullable(arrow_table)) -``` - -Replace it with the following block (same indentation — 8 spaces): - -```python - # ------------------------------------------------------------------ - # Establish or apply the canonical Arrow schema. - # - # The schema is a property of the *source*, not of any individual - # batch. A zero-row batch, a null-free batch, and a batch with nulls - # all represent data from the same source and must produce streams - # with identical nullability. - # - # Two paths: - # Declared — impl.schema() returned a non-None Schema at - # construction, so _tag_schema / _data_schema are populated. - # Derive the Arrow schema from the Python type annotations: - # T | None → nullable=True; plain T → nullable=False. - # No inference; no warning. - # Infer-once — impl.schema() returned None. Infer nullability from - # the first batch (which contains real data, so inference is - # meaningful) and cache the result. Emit a WARNING to prompt the - # caller to declare a schema. - # - # All subsequent batches are cast to the canonical schema by column - # name (order-safe). - # ------------------------------------------------------------------ - if self._canonical_arrow_schema is None: - if self._tag_schema is not None and self._data_schema is not None: - combined = {**dict(self._tag_schema), **dict(self._data_schema)} - self._canonical_arrow_schema = ( - self.data_context.type_converter.python_schema_to_arrow_schema(combined) - ) - else: - logger.warning( - "PollingSource %r: no schema declared via impl.schema(); " - "inferring nullability from first batch. Implement impl.schema() " - "to avoid schema drift on zero-row polls or null-free batches.", - self._source_id, - ) - self._canonical_arrow_schema = arrow_utils.infer_schema_nullable(arrow_table) - - canonical_nullable = {f.name: f.nullable for f in self._canonical_arrow_schema} - target_schema = pa.schema([ - pa.field(f.name, f.type, nullable=canonical_nullable.get(f.name, f.nullable)) - for f in arrow_table.schema - ]) - arrow_table = arrow_table.cast(target_schema) -``` - -The complete `_build_stream_from_df` method after the edit should be: - -```python - def _build_stream_from_df(self, df: pl.DataFrame) -> ArrowTableStream: - """Build an ``ArrowTableStream`` from a Polars DataFrame.""" - from orcapod.core.streams.arrow_table_stream import ArrowTableStream - - # Handle Object-dtype columns (same pattern as DataFrameSource) - object_columns = [c for c in df.columns if df[c].dtype == pl.Object] - if object_columns: - sub_table = self.data_context.type_converter.python_dicts_to_arrow_table( - df.select(object_columns).to_dicts() - ) - df = df.with_columns([pl.from_arrow(c) for c in sub_table]) - - df = polars_data_utils.drop_system_columns(df) - - arrow_table = df.to_arrow() - - if self._canonical_arrow_schema is None: - if self._tag_schema is not None and self._data_schema is not None: - combined = {**dict(self._tag_schema), **dict(self._data_schema)} - self._canonical_arrow_schema = ( - self.data_context.type_converter.python_schema_to_arrow_schema(combined) - ) - else: - logger.warning( - "PollingSource %r: no schema declared via impl.schema(); " - "inferring nullability from first batch. Implement impl.schema() " - "to avoid schema drift on zero-row polls or null-free batches.", - self._source_id, - ) - self._canonical_arrow_schema = arrow_utils.infer_schema_nullable(arrow_table) - - canonical_nullable = {f.name: f.nullable for f in self._canonical_arrow_schema} - target_schema = pa.schema([ - pa.field(f.name, f.type, nullable=canonical_nullable.get(f.name, f.nullable)) - for f in arrow_table.schema - ]) - arrow_table = arrow_table.cast(target_schema) - - builder = SourceStreamBuilder(self.data_context, self.orcapod_config) - result = builder.build( - arrow_table, - tag_columns=self._tag_columns, - source_id=self._source_id, - ) - return result.stream -``` - -- [ ] **Step 3: Run the four new regression tests** - -```bash -uv run pytest tests/test_channels/test_polling_source.py::TestPollingSourceZeroRowBatch -v -``` - -Expected: all four tests **PASS**. - -If any test fails, re-read the error and check the implementation — do not proceed to the full suite until these pass. - -- [ ] **Step 4: Run the full existing PollingSource test suite** - -```bash -uv run pytest tests/test_channels/test_polling_source.py -v -``` - -Expected: **all tests PASS**. Pay special attention to: - -- `TestPollingSourceSyncMode` — sync path also calls `_build_stream_from_df`; the canonical schema must not break it. -- `TestPollingSourceSchemaValidation` — declared-schema tests; the canonical schema path feeds the same `_tag_schema` / `_data_schema` already validated there. -- `TestPollingSourceAsyncMode::test_schema_mismatch_raises_on_column_change` — intentional schema mismatch must still raise. - -- [ ] **Step 5: Commit the implementation** - -```bash -git add src/orcapod/core/sources/polling_source.py -git commit -m "fix(polling_source): establish canonical Arrow schema once per source (ENG-952) - -Per-batch nullability re-inference via infer_schema_nullable caused -SchemaInconsistencyError whenever a poll returned a zero-row batch after -any batch that contained a null — a zero-row table always has -null_count == 0, so every field was inferred non-nullable. - -_build_stream_from_df now establishes _canonical_arrow_schema exactly -once: from impl.schema() Python type annotations when declared (T | None -→ nullable=True; plain T → nullable=False, no inference, no warning), or -from the first batch otherwise (with a WARNING-level log). All subsequent -batches are cast to the canonical schema by column name, so nullability is -stable across the source's lifetime regardless of per-batch null counts." -``` - ---- - -### Task 3: Add DESIGN_ISSUES.md entry and final checks - -**Files:** -- Modify: `DESIGN_ISSUES.md` -- Run: full channel test suite - -- [ ] **Step 1: Add PS4 entry to DESIGN_ISSUES.md** - -Locate the PS3 entry (ends around line 149). Insert the following block immediately after PS3's closing `---`: - -```markdown -### PS4 — `PollingSource` re-infers Arrow schema nullability per batch, crashing on zero-row polls -**Status:** resolved -**Severity:** high -**Issue:** ENG-952 - -`_build_stream_from_df` called `infer_schema_nullable` on every batch. A zero-row batch has -`null_count == 0` for all columns, so every field was inferred `nullable=False`. -`_validate_combining_schemas` then rejected the batch against the accumulated stream's nullable -schema, raising `SchemaInconsistencyError` — an `InputValidationError` that `async_iter_data` -re-raises immediately, killing the source. The common-path trigger: `WindowDiscoverySource` -in watch mode emits zero-row deltas once the discovery window is fully emitted (~144 -times/day per stage at a 10-minute interval). - -**Fix:** `_build_stream_from_df` now establishes `_canonical_arrow_schema` exactly once. -Declared-schema path (`impl.schema()` non-`None`): Arrow schema derived from Python type -annotations via `python_schema_to_arrow_schema` — `T | None` → `nullable=True`, plain `T` → -`nullable=False`; no inference, no warning. Infer-once path (`impl.schema()` returns `None`): -nullability inferred from the first batch with a `WARNING`-level log prompting the caller to -declare a schema. All subsequent batches are cast to the canonical schema by column name. -Per-batch re-inference is eliminated. - ---- -``` - -- [ ] **Step 2: Run the full test suite** - -```bash -uv run pytest tests/ -x -q -``` - -Expected: all tests pass. If any test other than the new ones fails, investigate before proceeding — the fix must not regress anything. - -- [ ] **Step 3: Commit DESIGN_ISSUES.md** - -```bash -git add DESIGN_ISSUES.md -git commit -m "docs: add PS4 to DESIGN_ISSUES.md — PollingSource zero-row batch crash (ENG-952)" -``` - ---- - -## Self-Review - -**Spec coverage:** - -| Spec requirement | Task | -|---|---| -| Canonical Arrow schema established once per source | Task 2 Step 2 | -| Declared-schema path: `T \| None` → `nullable=True`, no warning | Task 2 Step 2 (declared branch) | -| Infer-once path: infer from first batch, store, warn | Task 2 Step 2 (else branch) | -| Cast by column name (order-safe) | Task 2 Step 2 (`canonical_nullable` dict) | -| `_canonical_arrow_schema` attribute | Task 2 Step 1 | -| Regression test: zero-row after null streams cleanly | Task 1 (test 1) | -| Regression test: zero-row batch not accumulated | Task 1 (test 2) | -| Declared schema: no warning emitted | Task 1 (test 3) | -| Infer-once: exactly one warning | Task 1 (test 4) | -| DESIGN_ISSUES.md PS4 entry | Task 3 Step 1 | - -**Placeholder scan:** No TBDs, no "similar to" references, no missing code blocks. ✓ - -**Type consistency:** -- `_canonical_arrow_schema` named identically in Task 2 Steps 1 and 2. ✓ -- `canonical_nullable` dict built from `self._canonical_arrow_schema` and applied immediately in the same method. ✓ -- `FakeDynamicSource` and `_batch` referenced in Task 1 Test 4 — both defined earlier in the test file. ✓ -- `Schema`, `Cursor`, `PollingConfig`, `pa`, `logging` — all already imported at the top of the test file. ✓ diff --git a/superpowers/plans/2026-09-02-itl-638-stable-type-annotation-hashing.md b/superpowers/plans/2026-09-02-itl-638-stable-type-annotation-hashing.md deleted file mode 100644 index 5e9956a75..000000000 --- a/superpowers/plans/2026-09-02-itl-638-stable-type-annotation-hashing.md +++ /dev/null @@ -1,1245 +0,0 @@ -# Stable Type Annotation Hashing (ITL-638) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace module-path-based type serialization in function pod signature hashing with stable canonical names from `LogicalTypeRegistry`, so that internal module reorganizations (e.g. `op.File` moving between subpackages) no longer invalidate cached function pod signatures. - -**Architecture:** Three coordinated changes: (1) extend `canonical_annotation_str` in `hash_utils.py` to accept a `LogicalTypeRegistry` and resolve registered types to their stable `logical_type_name`; (2) fix `TypeObjectHandler` to use this registry for bare type objects; (3) fix `FunctionSignatureExtractor` to canonicalize both parameter annotation strings and return annotation through the same helper. Guarded by pre-fix golden-value fixtures that confirm the change only touches orcapod logical types (e.g. `op.File`) and leaves builtins untouched. - -**Tech Stack:** Python 3.11+, pytest via `uv run`, `orcapod.logical_types.registry.LogicalTypeRegistry`, `orcapod.hashing` - -**Branch:** `eywalker/itl-638-function-pod-signature-hashing-uses-full-type-import-paths` - ---- - -## File Map - -| File | Action | Purpose | -|------|--------|---------| -| `tests/test_hashing/generate_type_annotation_golden.py` | Create | Script to generate pre-fix golden hashes | -| `tests/test_hashing/hash_samples/type_annotation_golden.json` | Generate+commit | Frozen pre-fix hash values | -| `tests/test_hashing/test_type_annotation_golden.py` | Create | Golden comparison + diff regression tests | -| `src/orcapod/hashing/hash_utils.py` | Modify | Extend `canonical_annotation_str` with registry param | -| `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Modify | Fix `TypeObjectHandler` + wire registry in `register_builtin_python_type_handlers` | -| `src/orcapod/hashing/semantic_hashing/function_info_extractors.py` | Modify | Fix `FunctionSignatureExtractor` params + returns | - ---- - -### Task 1: Generate and commit pre-fix golden values - -**Files:** -- Create: `tests/test_hashing/generate_type_annotation_golden.py` -- Generate: `tests/test_hashing/hash_samples/type_annotation_golden.json` - -This task captures the current (broken) hash values before any fix. The JSON is committed as an immutable record. - -- [ ] **Step 1: Write the golden generator script** - -Create `tests/test_hashing/generate_type_annotation_golden.py`: - -```python -"""Generate pre-fix golden hash values for type annotation hashing. - -Run once (before the ITL-638 fix) with: - uv run python tests/test_hashing/generate_type_annotation_golden.py - -Outputs: tests/test_hashing/hash_samples/type_annotation_golden.json -""" -from __future__ import annotations - -import inspect -import json -import pathlib -import typing -from uuid import UUID - -import orcapod as op -from orcapod.hashing.defaults import get_default_semantic_hasher -from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionSignatureExtractor, -) -from orcapod.logical_types.file_type import File -from orcapod.logical_types.directory_type import Directory - -GOLDEN_PATH = pathlib.Path(__file__).parent / "hash_samples" / "type_annotation_golden.json" - -# --------------------------------------------------------------------------- -# Annotation types to hash individually (bare type objects) -# --------------------------------------------------------------------------- -ANNOTATION_CASES: dict[str, object] = { - # Builtins - "int": int, - "str": str, - "float": float, - "bytes": bytes, - # orcapod logical types - "op.File": op.File, - "op.Directory": op.Directory, - "op.Path": op.Path, - "op.UUID": UUID, - # Generic aliases - "list[int]": list[int], - "dict[str, int]": dict[str, int], - "list[op.File]": list[op.File], - "dict[str, op.File]": dict[str, op.File], - # Unions - "int | str": int | str, - "op.File | None": op.File | None, - "Optional[op.File]": typing.Optional[op.File], -} - -# --------------------------------------------------------------------------- -# Functions whose full hash_object(func) and extract_function_info output -# are both captured. -# --------------------------------------------------------------------------- - -def fn_no_annotations(): - return None - -def fn_builtin_param(x: int, y: str) -> float: - return float(x) - -def fn_orcapod_param(f: op.File) -> str: - return str(f) - -def fn_orcapod_return(s: str) -> op.File: - return op.File(s) # type: ignore[arg-type] - -def fn_generic_orcapod(files: list[op.File]) -> list[str]: - return [] - -def fn_union_orcapod(f: op.File | None) -> op.File | None: - return f - -def fn_mixed(f: op.File, n: int) -> op.Directory: - return op.Directory(str(f)) # type: ignore[arg-type] - -FUNCTION_CASES: dict[str, object] = { - "fn_no_annotations": fn_no_annotations, - "fn_builtin_param": fn_builtin_param, - "fn_orcapod_param": fn_orcapod_param, - "fn_orcapod_return": fn_orcapod_return, - "fn_generic_orcapod": fn_generic_orcapod, - "fn_union_orcapod": fn_union_orcapod, - "fn_mixed": fn_mixed, -} - - -def main() -> None: - hasher = get_default_semantic_hasher() - extractor = FunctionSignatureExtractor(include_module=True, include_defaults=True) - - result: dict = {"annotation_hashes": {}, "function_info_hashes": {}, "function_object_hashes": {}} - - # Hash bare annotations through TypeObjectHandler - for key, ann in ANNOTATION_CASES.items(): - result["annotation_hashes"][key] = hasher.hash_object(ann).to_string() - - # Hash FunctionSignatureExtractor output dict, and full function object - for key, func in FUNCTION_CASES.items(): - info = extractor.extract_function_info(func) - result["function_info_hashes"][key] = hasher.hash_object(info).to_string() - result["function_object_hashes"][key] = hasher.hash_object(func).to_string() - - GOLDEN_PATH.parent.mkdir(parents=True, exist_ok=True) - GOLDEN_PATH.write_text(json.dumps(result, indent=2) + "\n") - print(f"Wrote golden values to {GOLDEN_PATH}") - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 2: Run the generator to produce the golden JSON** - -```bash -cd /path/to/orcapod-python -uv run python tests/test_hashing/generate_type_annotation_golden.py -``` - -Expected output: -``` -Wrote golden values to tests/test_hashing/hash_samples/type_annotation_golden.json -``` - -- [ ] **Step 3: Inspect the golden JSON to verify it looks reasonable** - -```bash -uv run python -c " -import json, pathlib -d = json.loads(pathlib.Path('tests/test_hashing/hash_samples/type_annotation_golden.json').read_text()) -for section, entries in d.items(): - print(f'\\n=== {section} ===') - for k, v in entries.items(): - print(f' {k}: {v}') -" -``` - -Verify that: -- `op.File`, `op.Directory`, `op.Path`, `op.UUID` annotation hashes contain the full module path (e.g. the hash input is `"type:orcapod.logical_types.file_type.File"`) -- Builtin hashes (`int`, `str`, etc.) are present - -- [ ] **Step 4: Commit the generator script and the golden JSON** - -```bash -git add tests/test_hashing/generate_type_annotation_golden.py -git add tests/test_hashing/hash_samples/type_annotation_golden.json -git commit -m "test(hashing): add pre-fix golden hash values for type annotation hashing (ITL-638)" -``` - ---- - -### Task 2: Write golden-comparison test skeleton (failing) - -**Files:** -- Create: `tests/test_hashing/test_type_annotation_golden.py` - -This test loads the golden JSON and will assert (post-fix) that builtins are unchanged and orcapod types changed to canonical names. Writing it now (before the fix) means the "changed" assertions will fail until the fix lands. - -- [ ] **Step 1: Write the test file** - -Create `tests/test_hashing/test_type_annotation_golden.py`: - -```python -"""Golden-value regression tests for type annotation hashing (ITL-638). - -Two test classes: - TestGoldenStability -- builtins must be UNCHANGED after the fix. - TestGoldenCanonical -- orcapod logical types must produce NEW canonical hashes. - -The golden JSON was generated pre-fix by generate_type_annotation_golden.py. -""" -from __future__ import annotations - -import json -import pathlib -import typing -from uuid import UUID - -import pytest - -import orcapod as op -from orcapod.hashing.defaults import get_default_semantic_hasher -from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionSignatureExtractor, -) - -GOLDEN_PATH = ( - pathlib.Path(__file__).parent / "hash_samples" / "type_annotation_golden.json" -) - -# --------------------------------------------------------------------------- -# The same annotation and function cases as the generator — must stay in sync. -# --------------------------------------------------------------------------- - -ANNOTATION_CASES: dict[str, object] = { - "int": int, - "str": str, - "float": float, - "bytes": bytes, - "op.File": op.File, - "op.Directory": op.Directory, - "op.Path": op.Path, - "op.UUID": UUID, - "list[int]": list[int], - "dict[str, int]": dict[str, int], - "list[op.File]": list[op.File], - "dict[str, op.File]": dict[str, op.File], - "int | str": int | str, - "op.File | None": op.File | None, - "Optional[op.File]": typing.Optional[op.File], -} - -# Annotation keys that are expected to change after the fix. -# Any key not in this set must have an UNCHANGED hash. -EXPECTED_CHANGED_KEYS: frozenset[str] = frozenset({ - "op.File", - "op.Directory", - "op.Path", - "op.UUID", - "list[op.File]", - "dict[str, op.File]", - "op.File | None", - "Optional[op.File]", -}) - - -def fn_no_annotations(): - return None - -def fn_builtin_param(x: int, y: str) -> float: - return float(x) - -def fn_orcapod_param(f: op.File) -> str: - return str(f) - -def fn_orcapod_return(s: str) -> op.File: - return op.File(s) # type: ignore[arg-type] - -def fn_generic_orcapod(files: list[op.File]) -> list[str]: - return [] - -def fn_union_orcapod(f: op.File | None) -> op.File | None: - return f - -def fn_mixed(f: op.File, n: int) -> op.Directory: - return op.Directory(str(f)) # type: ignore[arg-type] - -FUNCTION_CASES: dict[str, object] = { - "fn_no_annotations": fn_no_annotations, - "fn_builtin_param": fn_builtin_param, - "fn_orcapod_param": fn_orcapod_param, - "fn_orcapod_return": fn_orcapod_return, - "fn_generic_orcapod": fn_generic_orcapod, - "fn_union_orcapod": fn_union_orcapod, - "fn_mixed": fn_mixed, -} - -# Functions expected to have different hashes after the fix. -EXPECTED_CHANGED_FUNCTIONS: frozenset[str] = frozenset({ - "fn_orcapod_param", - "fn_orcapod_return", - "fn_generic_orcapod", - "fn_union_orcapod", - "fn_mixed", -}) - - -@pytest.fixture(scope="module") -def golden() -> dict: - assert GOLDEN_PATH.exists(), ( - f"Golden file not found: {GOLDEN_PATH}. " - "Run generate_type_annotation_golden.py first." - ) - return json.loads(GOLDEN_PATH.read_text()) - - -@pytest.fixture(scope="module") -def hasher(): - return get_default_semantic_hasher() - - -@pytest.fixture(scope="module") -def extractor(): - return FunctionSignatureExtractor(include_module=True, include_defaults=True) - - -class TestGoldenStability: - """Builtins and non-orcapod annotations must hash identically before and after the fix.""" - - def test_builtin_annotation_hashes_unchanged(self, golden, hasher): - stable_keys = {k for k in ANNOTATION_CASES if k not in EXPECTED_CHANGED_KEYS} - mismatches = {} - for key in stable_keys: - ann = ANNOTATION_CASES[key] - current = hasher.hash_object(ann).to_string() - expected = golden["annotation_hashes"][key] - if current != expected: - mismatches[key] = {"expected": expected, "current": current} - assert not mismatches, ( - f"Unexpected hash changes in stable annotations:\n" - + "\n".join(f" {k}: {v}" for k, v in mismatches.items()) - ) - - def test_builtin_function_hashes_unchanged(self, golden, hasher, extractor): - stable_fns = {k for k in FUNCTION_CASES if k not in EXPECTED_CHANGED_FUNCTIONS} - mismatches = {} - for key in stable_fns: - func = FUNCTION_CASES[key] - info = extractor.extract_function_info(func) - current = hasher.hash_object(info).to_string() - expected = golden["function_info_hashes"][key] - if current != expected: - mismatches[key] = {"expected": expected, "current": current} - assert not mismatches, ( - f"Unexpected hash changes in stable functions:\n" - + "\n".join(f" {k}: {v}" for k, v in mismatches.items()) - ) - - -class TestGoldenCanonical: - """orcapod logical types must produce NEW hashes (canonical name, not module path).""" - - def test_orcapod_annotation_hashes_changed(self, golden, hasher): - """After the fix, orcapod type annotation hashes must DIFFER from golden.""" - unchanged = {} - for key in EXPECTED_CHANGED_KEYS: - if key not in ANNOTATION_CASES: - continue - ann = ANNOTATION_CASES[key] - current = hasher.hash_object(ann).to_string() - expected = golden["annotation_hashes"][key] - if current == expected: - unchanged[key] = current - assert not unchanged, ( - f"Expected these annotation hashes to change after the fix, but they didn't:\n" - + "\n".join(f" {k}: {v}" for k, v in unchanged.items()) - ) - - def test_orcapod_function_hashes_changed(self, golden, hasher, extractor): - """After the fix, functions using orcapod types must DIFFER from golden.""" - unchanged = {} - for key in EXPECTED_CHANGED_FUNCTIONS: - func = FUNCTION_CASES[key] - info = extractor.extract_function_info(func) - current = hasher.hash_object(info).to_string() - expected = golden["function_info_hashes"][key] - if current == expected: - unchanged[key] = current - assert not unchanged, ( - f"Expected these function hashes to change after the fix, but they didn't:\n" - + "\n".join(f" {k}: {v}" for k, v in unchanged.items()) - ) -``` - -- [ ] **Step 2: Verify the stability tests pass (pre-fix) and canonical tests fail** - -```bash -uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestGoldenStability -v -``` - -Expected: All PASS (hashes match golden before any fix) - -```bash -uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestGoldenCanonical -v -``` - -Expected: All FAIL (hashes still match golden — the fix hasn't landed yet) - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_hashing/test_type_annotation_golden.py -git commit -m "test(hashing): add golden-diff regression tests for type annotation canonicalization (ITL-638)" -``` - ---- - -### Task 3: Extend `canonical_annotation_str` with registry support - -**Files:** -- Modify: `src/orcapod/hashing/hash_utils.py` -- Test: `tests/test_hashing/test_hash_utils.py` - -The existing `canonical_annotation_str` only handles union ordering. We extend it to accept an optional `LogicalTypeRegistry` and also handle generic alias recursion. - -- [ ] **Step 1: Write failing tests for the extended function** - -Add to `tests/test_hashing/test_hash_utils.py` (append to existing file — do not replace existing tests): - -```python -# --------------------------------------------------------------------------- -# Tests for canonical_annotation_str with registry (ITL-638) -# --------------------------------------------------------------------------- -import typing -from uuid import UUID - -import orcapod as op -from orcapod.contexts import get_default_logical_type_registry -from orcapod.hashing.hash_utils import canonical_annotation_str - - -class TestCanonicalAnnotationStrWithRegistry: - """canonical_annotation_str resolves registered logical types to stable names.""" - - @pytest.fixture - def registry(self): - return get_default_logical_type_registry() - - def test_builtin_type_unchanged(self, registry): - assert canonical_annotation_str(int, registry) == "int" - - def test_builtin_str_unchanged(self, registry): - assert canonical_annotation_str(str, registry) == "str" - - def test_registered_type_uses_logical_name(self, registry): - result = canonical_annotation_str(op.File, registry) - assert result == "orcapod.file" - - def test_registered_directory_uses_logical_name(self, registry): - result = canonical_annotation_str(op.Directory, registry) - assert result == "orcapod.directory" - - def test_registered_path_uses_logical_name(self, registry): - import pathlib - result = canonical_annotation_str(pathlib.Path, registry) - assert result == "orcapod.path" - - def test_registered_uuid_uses_logical_name(self, registry): - result = canonical_annotation_str(UUID, registry) - assert result == "orcapod.uuid" - - def test_generic_list_of_registered_type(self, registry): - result = canonical_annotation_str(list[op.File], registry) - assert result == "list[orcapod.file]" - - def test_generic_dict_with_registered_value(self, registry): - result = canonical_annotation_str(dict[str, op.File], registry) - assert result == "dict[str, orcapod.file]" - - def test_union_with_registered_type(self, registry): - result = canonical_annotation_str(op.File | None, registry) - # Members sorted; NoneType sorts before orcapod.file - assert result == "NoneType | orcapod.file" - - def test_optional_registered_type(self, registry): - result = canonical_annotation_str(typing.Optional[op.File], registry) - assert result == "NoneType | orcapod.file" - - def test_no_registry_fallback(self): - """Without registry, behaviour is identical to the existing function.""" - import inspect - result = canonical_annotation_str(op.File, None) - assert result == inspect.formatannotation(op.File) - - def test_stable_across_calls(self, registry): - r1 = canonical_annotation_str(op.File, registry) - r2 = canonical_annotation_str(op.File, registry) - assert r1 == r2 -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_hash_utils.py::TestCanonicalAnnotationStrWithRegistry -v -``` - -Expected: All FAIL (function not yet extended) - -- [ ] **Step 3: Extend `canonical_annotation_str` in `hash_utils.py`** - -Replace the existing `canonical_annotation_str` function (currently lines 35-58) with this extended version. Add the `TYPE_CHECKING` guard at the top of the file if not already present: - -```python -# At top of file, inside the existing imports block: -from __future__ import annotations -import inspect -import types as _types -import typing -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from orcapod.logical_types.registry import LogicalTypeRegistry -``` - -Replace the `canonical_annotation_str` function body: - -```python -def canonical_annotation_str( - annotation: object, - registry: "LogicalTypeRegistry | None" = None, -) -> str: - """Return a stable, canonical string for a type annotation. - - Resolves types registered in *registry* (e.g. ``orcapod.File``) to their - stable ``logical_type_name`` (e.g. ``"orcapod.file"``) so that internal - module relocations do not change the string representation. - - For union types (both PEP 604 ``X | Y`` and ``typing.Union[X, Y]``), - members are sorted byte-wise so that ``str | Path`` and ``Path | str`` - produce the same canonical string. - - For generic aliases (``list[X]``, ``dict[K, V]``), args are recursed with - the same registry so nested orcapod types are also canonicalized. - - Non-union, non-generic types not found in the registry fall through to - ``inspect.formatannotation``, preserving existing behaviour exactly. - - Args: - annotation: A type annotation object. - registry: Optional ``LogicalTypeRegistry``. When provided, registered - logical types resolve to their stable ``logical_type_name``. - - Returns: - A canonical string representation. - """ - # Registered logical type: use stable canonical name (e.g. "orcapod.file") - if registry is not None and isinstance(annotation, type): - lt = registry.get_by_python_type(annotation) - if lt is not None: - return lt.logical_type_name - - # Union types (PEP 604 X | Y and typing.Union): sort members for order-independence - if is_union_annotation(annotation): - args = getattr(annotation, "__args__", ()) or () - member_strs = sorted(canonical_annotation_str(a, registry) for a in args) - return " | ".join(member_strs) - - # Generic aliases (list[X], dict[K, V], typing.List[X], etc.): recurse args - origin = getattr(annotation, "__origin__", None) - if origin is not None and not is_union_annotation(annotation): - args = getattr(annotation, "__args__", None) or () - origin_str = canonical_annotation_str(origin, registry) - if args: - args_str = ", ".join(canonical_annotation_str(a, registry) for a in args) - return f"{origin_str}[{args_str}]" - return origin_str - - return inspect.formatannotation(annotation) -``` - -- [ ] **Step 4: Run the new tests to confirm they pass** - -```bash -uv run pytest tests/test_hashing/test_hash_utils.py::TestCanonicalAnnotationStrWithRegistry -v -``` - -Expected: All PASS - -- [ ] **Step 5: Run the full `test_hash_utils.py` to confirm no regressions** - -```bash -uv run pytest tests/test_hashing/test_hash_utils.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/hash_utils.py tests/test_hashing/test_hash_utils.py -git commit -m "feat(hashing): extend canonical_annotation_str to resolve registered logical types (ITL-638)" -``` - ---- - -### Task 4: Fix `TypeObjectHandler` to use registry lookup - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` -- Test: `tests/test_hashing/test_semantic_hasher.py` - -- [ ] **Step 1: Write failing tests for registry-aware `TypeObjectHandler`** - -Add to `tests/test_hashing/test_semantic_hasher.py` (append after the existing `TestTypeObjectHandler` class): - -```python -class TestTypeObjectHandlerWithRegistry: - """TypeObjectHandler uses stable canonical names for registered logical types.""" - - @pytest.fixture - def registry(self): - from orcapod.contexts import get_default_logical_type_registry - return get_default_logical_type_registry() - - @pytest.fixture - def handler(self, registry): - from orcapod.hashing.semantic_hashing.builtin_handlers import TypeObjectHandler - return TypeObjectHandler(logical_type_registry=registry) - - def test_registered_type_returns_canonical_name(self, handler, hasher): - import orcapod as op - result = handler.handle(op.File, hasher) - assert result == "type:orcapod.file" - - def test_registered_directory_returns_canonical_name(self, handler, hasher): - import orcapod as op - result = handler.handle(op.Directory, hasher) - assert result == "type:orcapod.directory" - - def test_unregistered_type_falls_back_to_module_qualname(self, handler, hasher): - result = handler.handle(int, hasher) - assert result == "type:builtins.int" - - def test_custom_class_falls_back_to_module_qualname(self, handler, hasher): - class _Local: - pass - result = handler.handle(_Local, hasher) - assert "type:" in result - assert "_Local" in result - - def test_hash_stable_across_calls(self, registry): - """Hashing op.File twice with registry produces identical ContentHash.""" - from orcapod.hashing.semantic_hashing.builtin_handlers import ( - TypeObjectHandler, - register_builtin_python_type_handlers, - ) - from orcapod.hashing.semantic_hashing.type_handler_registry import ( - PythonTypeHandlerRegistry, - ) - from orcapod.hashing.semantic_hashing.semantic_hasher import ( - SemanticAwarePythonHasher, - ) - import orcapod as op - - reg = PythonTypeHandlerRegistry() - register_builtin_python_type_handlers(reg, logical_type_registry=registry) - h = SemanticAwarePythonHasher(hasher_id="test_v1", type_handler_registry=reg) - assert h.hash_object(op.File) == h.hash_object(op.File) - - def test_simulated_module_relocation_stable(self, registry): - """Relocating a class's __module__ does not change the hash if its - logical_type_name is unchanged in the registry. - """ - from orcapod.hashing.semantic_hashing.builtin_handlers import ( - register_builtin_python_type_handlers, - ) - from orcapod.hashing.semantic_hashing.type_handler_registry import ( - PythonTypeHandlerRegistry, - ) - from orcapod.hashing.semantic_hashing.semantic_hasher import ( - SemanticAwarePythonHasher, - ) - import orcapod as op - - reg = PythonTypeHandlerRegistry() - register_builtin_python_type_handlers(reg, logical_type_registry=registry) - h = SemanticAwarePythonHasher(hasher_id="test_v1", type_handler_registry=reg) - - hash_before = h.hash_object(op.File) - - # Simulate module relocation by temporarily patching __module__ - original_module = op.File.__module__ - try: - op.File.__module__ = "orcapod.extension_types.file_type" - hash_after = h.hash_object(op.File) - finally: - op.File.__module__ = original_module - - assert hash_before == hash_after, ( - "Hash changed when op.File.__module__ was altered — " - "registry lookup is not being used." - ) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_semantic_hasher.py::TestTypeObjectHandlerWithRegistry -v -``` - -Expected: All FAIL (`TypeObjectHandler.__init__` does not accept `logical_type_registry` yet) - -- [ ] **Step 3: Update `TypeObjectHandler` in `builtin_handlers.py`** - -Replace the `TypeObjectHandler` class (currently lines 83-96): - -```python -class TypeObjectHandler: - """Hasher for type objects (classes passed as values). - - Resolves types registered in *logical_type_registry* to their stable - ``logical_type_name`` (e.g. ``"type:orcapod.file"`` for ``op.File``). - Falls back to ``"type:."`` for unregistered types. - - Args: - logical_type_registry: Optional ``LogicalTypeRegistry``. When ``None``, - the default context's registry is resolved lazily at call time, - following the same pattern as ``ArrowTableHandler``. - """ - - def __init__(self, logical_type_registry: Any = None) -> None: - self._logical_type_registry = logical_type_registry - - def _get_registry(self) -> Any: - if self._logical_type_registry is not None: - return self._logical_type_registry - from orcapod.contexts import get_default_context - return get_default_context().logical_type_registry - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: - if not isinstance(obj, type): - raise TypeError( - f"TypeObjectHandler: expected a type/class, got {type(obj)!r}" - ) - registry = self._get_registry() - lt = registry.get_by_python_type(obj) - if lt is not None: - return f"type:{lt.logical_type_name}" - module: str = obj.__module__ or "" - qualname: str = obj.__qualname__ - return f"type:{module}.{qualname}" -``` - -- [ ] **Step 4: Run the new tests to confirm they pass** - -```bash -uv run pytest tests/test_hashing/test_semantic_hasher.py::TestTypeObjectHandlerWithRegistry -v -``` - -Expected: All PASS - -- [ ] **Step 5: Run the full `test_semantic_hasher.py` to confirm no regressions** - -```bash -uv run pytest tests/test_hashing/test_semantic_hasher.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py \ - tests/test_hashing/test_semantic_hasher.py -git commit -m "feat(hashing): make TypeObjectHandler resolve logical types to canonical names (ITL-638)" -``` - ---- - -### Task 5: Fix `FunctionSignatureExtractor` — consistent canonicalization for params and returns - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/function_info_extractors.py` -- Test: `tests/test_hashing/test_function_info_extractors.py` - -Currently: params embed annotation string via `str(param)` (module path), returns stores raw type object. After fix: both use `canonical_annotation_str(annotation, registry)`. - -- [ ] **Step 1: Write failing tests for registry-aware `FunctionSignatureExtractor`** - -Add to `tests/test_hashing/test_function_info_extractors.py` (append after existing tests): - -```python -class TestFunctionSignatureExtractorWithRegistry: - """FunctionSignatureExtractor canonicalizes both param and return annotations.""" - - @pytest.fixture - def registry(self): - from orcapod.contexts import get_default_logical_type_registry - return get_default_logical_type_registry() - - @pytest.fixture - def extractor(self, registry): - from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionSignatureExtractor, - ) - return FunctionSignatureExtractor( - include_module=True, - include_defaults=True, - logical_type_registry=registry, - ) - - def test_return_annotation_is_canonical_string(self, extractor): - """parts['returns'] must be a string, not a type object.""" - import orcapod as op - - def fn(s: str) -> op.File: - ... - - info = extractor.extract_function_info(fn) - assert isinstance(info["returns"], str), ( - f"Expected str, got {type(info['returns'])}: {info['returns']!r}" - ) - assert info["returns"] == "orcapod.file" - - def test_param_annotation_is_canonical_string(self, extractor): - """Parameter annotation in params string uses logical_type_name.""" - import orcapod as op - - def fn(f: op.File) -> str: - ... - - info = extractor.extract_function_info(fn) - assert "orcapod.file" in info["params"], ( - f"Expected 'orcapod.file' in params, got: {info['params']!r}" - ) - assert "logical_types" not in info["params"], ( - f"Module path leaked into params: {info['params']!r}" - ) - - def test_generic_param_annotation_canonical(self, extractor): - """list[op.File] in a parameter is canonicalized.""" - import orcapod as op - - def fn(files: list[op.File]) -> str: - ... - - info = extractor.extract_function_info(fn) - assert "orcapod.file" in info["params"] - assert "logical_types" not in info["params"] - - def test_union_return_annotation_canonical(self, extractor): - """op.File | None return is canonicalized.""" - import orcapod as op - - def fn(s: str) -> op.File | None: - ... - - info = extractor.extract_function_info(fn) - assert isinstance(info["returns"], str) - assert "orcapod.file" in info["returns"] - assert "logical_types" not in info["returns"] - - def test_builtin_annotations_unchanged(self, extractor): - """Functions with only builtin annotations are unaffected.""" - def fn(x: int, y: str) -> float: - ... - - info = extractor.extract_function_info(fn) - assert "int" in info["params"] - assert "str" in info["params"] - assert info["returns"] == "float" - - def test_return_and_param_use_same_canonical_form(self, extractor): - """op.File in param and op.File as return use the same canonical string.""" - import orcapod as op - - def fn_param(f: op.File) -> str: - ... - - def fn_return(s: str) -> op.File: - ... - - info_param = extractor.extract_function_info(fn_param) - info_return = extractor.extract_function_info(fn_return) - - # Both should contain "orcapod.file" in their respective places - assert "orcapod.file" in info_param["params"] - assert info_return["returns"] == "orcapod.file" - - def test_simulated_relocation_stable(self, extractor): - """Patching __module__ on op.File does not change the extracted info.""" - import orcapod as op - - def fn(f: op.File) -> op.File: - ... - - info_before = extractor.extract_function_info(fn) - - original = op.File.__module__ - try: - op.File.__module__ = "orcapod.extension_types.file_type" - info_after = extractor.extract_function_info(fn) - finally: - op.File.__module__ = original - - assert info_before["params"] == info_after["params"] - assert info_before["returns"] == info_after["returns"] - - def test_no_registry_behaves_like_before(self): - """Without registry, returns is still a type object (pre-fix behavior).""" - import orcapod as op - from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionSignatureExtractor, - ) - - extractor_no_reg = FunctionSignatureExtractor( - include_module=True, include_defaults=True, logical_type_registry=None - ) - - def fn(s: str) -> op.File: - ... - - info = extractor_no_reg.extract_function_info(fn) - # Without registry, returns is the raw type object (legacy behavior) - assert isinstance(info["returns"], type) -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -```bash -uv run pytest tests/test_hashing/test_function_info_extractors.py::TestFunctionSignatureExtractorWithRegistry -v -``` - -Expected: All FAIL (`logical_type_registry` param not accepted, `returns` is a type object not a string) - -- [ ] **Step 3: Update `FunctionSignatureExtractor` in `function_info_extractors.py`** - -Replace the `FunctionSignatureExtractor` class entirely: - -```python -class FunctionSignatureExtractor: - """Extractor that uses the function signature for information extraction. - - Canonicalizes type annotations for both parameters and the return type - using ``canonical_annotation_str``. When *logical_type_registry* is - provided (or resolved lazily from the default context), registered - orcapod types (e.g. ``op.File``) are serialized to their stable - ``logical_type_name`` (e.g. ``"orcapod.file"``) rather than to their - fully-qualified import path. - - This ensures that internal module reorganizations (e.g. moving - ``File`` between subpackages) do not invalidate function pod caches. - - Args: - include_module: Include ``func.__module__`` in the extracted info. - include_defaults: Include default parameter values. - logical_type_registry: Optional ``LogicalTypeRegistry``. When - ``None``, the default context's registry is resolved lazily at - call time, following the same pattern as ``ArrowTableHandler``. - """ - - def __init__( - self, - include_module: bool = True, - include_defaults: bool = True, - logical_type_registry: Any = None, - ) -> None: - self.include_module = include_module - self.include_defaults = include_defaults - self._logical_type_registry = logical_type_registry - - def _get_registry(self) -> Any: - if self._logical_type_registry is not None: - return self._logical_type_registry - from orcapod.contexts import get_default_context - return get_default_context().logical_type_registry - - # FIXME: Fix this implementation!! - # BUG: Currently this is not using the input_types and output_types parameters - def extract_function_info( - self, - func: Callable[..., Any], - function_name: str | None = None, - input_typespec: Schema | None = None, - output_typespec: Schema | None = None, - ) -> dict[str, Any]: - if not callable(func): - raise TypeError("Provided object is not callable") - - # Use eval_str=True so that string annotations produced by - # ``from __future__ import annotations`` (PEP 563) are resolved to live - # type objects before canonicalization. - try: - sig = inspect.signature(func, eval_str=True) - except (NameError, TypeError, AttributeError, SyntaxError): - sig = inspect.signature(func) - - registry = self._get_registry() - parts: dict[str, Any] = {} - - if self.include_module and hasattr(func, "__module__"): - parts["module"] = func.__module__ - - parts["name"] = function_name or func.__name__ - - param_strs = [] - for name, param in sig.parameters.items(): - param_str = str(param) - annotation = param.annotation - if annotation is not inspect.Parameter.empty: - old_ann = inspect.formatannotation(annotation) - new_ann = canonical_annotation_str(annotation, registry) - if old_ann != new_ann: - # Replace ": " with ": " (first occurrence). - # The ": " prefix distinguishes the annotation from the default. - param_str = param_str.replace(f": {old_ann}", f": {new_ann}", 1) - if not self.include_defaults and "=" in param_str: - param_str = param_str.split("=")[0].strip() - param_strs.append(param_str) - - parts["params"] = ", ".join(param_strs) - - if sig.return_annotation is not inspect.Signature.empty: - # Store as canonical string (not raw type object) for consistency - # with the params representation and for registry-stable hashing. - parts["returns"] = canonical_annotation_str( - sig.return_annotation, registry - ) - - return parts -``` - -Add the import of `canonical_annotation_str` at the top of `function_info_extractors.py` (it's already imported from `hash_utils`; verify the import line includes it): - -```python -from orcapod.hashing.hash_utils import canonical_annotation_str, is_union_annotation -``` - -Also add `Any` to the imports if not present: - -```python -from typing import Any, Literal -``` - -- [ ] **Step 4: Run the new tests to confirm they pass** - -```bash -uv run pytest tests/test_hashing/test_function_info_extractors.py::TestFunctionSignatureExtractorWithRegistry -v -``` - -Expected: All PASS - -- [ ] **Step 5: Run the full `test_function_info_extractors.py` to confirm no regressions** - -```bash -uv run pytest tests/test_hashing/test_function_info_extractors.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/function_info_extractors.py \ - tests/test_hashing/test_function_info_extractors.py -git commit -m "feat(hashing): canonicalize param and return annotations in FunctionSignatureExtractor (ITL-638)" -``` - ---- - -### Task 6: Wire registry into `register_builtin_python_type_handlers` - -**Files:** -- Modify: `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` - -Pass the `LogicalTypeRegistry` through to both `TypeObjectHandler` and `FunctionSignatureExtractor` so callers who already have a registry don't rely on lazy context lookup. - -- [ ] **Step 1: Update `register_builtin_python_type_handlers` signature and body** - -In `builtin_handlers.py`, update the function signature (currently around line 415): - -```python -def register_builtin_python_type_handlers( - registry: "HandlerRegistryProtocol", - file_hasher: Any = None, - function_info_extractor: Any = None, - arrow_hasher: "ArrowHasherProtocol | None" = None, - directory_hasher: Any = None, - logical_type_registry: Any = None, -) -> None: - """Register all built-in semantic hashers into *registry*. - - Args: - registry: The ``HandlerRegistryProtocol`` instance to populate. - file_hasher: Optional ``FileContentHasherProtocol`` for file content hashing. - Defaults to ``FileHasher(sha256)``. - function_info_extractor: Optional ``FunctionInfoExtractorProtocol``. - Defaults to ``FunctionSignatureExtractor``. - arrow_hasher: Optional ``ArrowHasherProtocol`` for nested table hashing. - When ``None``, lazy resolution via the default context is used. - directory_hasher: Optional ``DirectoryHasherProtocol`` for directory tree hashing. - Defaults to ``BasicDirectoryHasher(sha256)``. - logical_type_registry: Optional ``LogicalTypeRegistry`` forwarded to - ``TypeObjectHandler`` and ``FunctionSignatureExtractor`` for stable - canonical-name resolution. When ``None``, both handlers resolve the - default context's registry lazily at call time. - """ -``` - -Then inside the function body, update the two relevant instantiation lines: - -```python - # Replace the existing FunctionSignatureExtractor instantiation: - if function_info_extractor is None: - from orcapod.hashing.semantic_hashing.function_info_extractors import ( - FunctionSignatureExtractor, - ) - function_info_extractor = FunctionSignatureExtractor( - include_module=True, - include_defaults=True, - logical_type_registry=logical_type_registry, - ) - - # Replace the existing TypeObjectHandler registration line: - registry.register(type, TypeObjectHandler(logical_type_registry=logical_type_registry)) -``` - -All other lines in the function body remain unchanged. - -- [ ] **Step 2: Run the full hashing test suite to confirm nothing is broken** - -```bash -uv run pytest tests/test_hashing/ -v --tb=short 2>&1 | tail -40 -``` - -Expected: All PASS - -- [ ] **Step 3: Commit** - -```bash -git add src/orcapod/hashing/semantic_hashing/builtin_handlers.py -git commit -m "feat(hashing): thread LogicalTypeRegistry through register_builtin_python_type_handlers (ITL-638)" -``` - ---- - -### Task 7: Run golden-diff tests and verify impact is limited to orcapod types - -At this point all three changes are live. The golden-diff tests from Task 2 should now behave correctly. - -- [ ] **Step 1: Run `TestGoldenStability` — builtins must be unchanged** - -```bash -uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestGoldenStability -v -``` - -Expected: All PASS (builtin hashes unchanged — `int`, `str`, `list[int]`, `dict[str, int]`, `int | str`, pure-builtin functions) - -- [ ] **Step 2: Run `TestGoldenCanonical` — orcapod types must have changed** - -```bash -uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestGoldenCanonical -v -``` - -Expected: All PASS (orcapod type hashes differ from golden — `op.File`, `op.Directory`, `op.Path`, `op.UUID` and any function using them) - -- [ ] **Step 3: If any `TestGoldenStability` test fails, investigate before proceeding** - -A failure in `TestGoldenStability` means an unintended hash change. Diagnose by printing the `info` dict from `extract_function_info` for the failing case and comparing to what the golden fixture captured. Fix the root cause; do not adjust the golden file. - -- [ ] **Step 4: Run the full test suite** - -```bash -uv run pytest tests/ -v --tb=short -q 2>&1 | tail -60 -``` - -Expected: All PASS - -- [ ] **Step 5: Commit the final test results note** - -No code change needed here; just record that validation passed. - ---- - -### Task 8: Create PR - -- [ ] **Step 1: Push the branch** - -```bash -git push -u origin eywalker/itl-638-function-pod-signature-hashing-uses-full-type-import-paths -``` - -- [ ] **Step 2: Open the PR** - -```bash -gh pr create \ - --base main \ - --title "fix(hashing): use stable canonical names for type annotation hashing (ITL-638)" \ - --body "$(cat <<'EOF' -## Summary - -Fixes ITL-638. Function pod signature hashing previously encoded types by their fully-qualified import path (e.g. `type:orcapod.logical_types.file_type.File`), causing cache invalidation whenever a type moved between internal modules. - -Changes: -- **`hash_utils.py`**: Extended `canonical_annotation_str` with an optional `LogicalTypeRegistry` parameter. Registered orcapod types resolve to their stable `logical_type_name` (e.g. `"orcapod.file"`). Generic aliases (`list[op.File]`) and union types (`op.File | None`) are recursed so nested types are also canonicalized. -- **`builtin_handlers.py`**: `TypeObjectHandler` now accepts a `LogicalTypeRegistry` (lazy fallback to default context). Registered types return `"type:orcapod.file"` instead of `"type:orcapod.logical_types.file_type.File"`. -- **`function_info_extractors.py`**: `FunctionSignatureExtractor` accepts a `LogicalTypeRegistry` and uses `canonical_annotation_str` for **both** parameter annotation strings and the return annotation (previously stored as a raw type object — now consistently a canonical string). -- **`register_builtin_python_type_handlers`**: Threads `logical_type_registry` through to both handlers. - -**Backward compatibility:** One-time cache invalidation for any function pod whose signature contained `op.File`, `op.Directory`, `op.Path`, or `op.UUID`. This is accepted (pre-v0.1.0, per CLAUDE.md). Builtins and non-orcapod types are unaffected (verified by golden-diff tests). - -## Test plan - -- [ ] `TestGoldenStability` passes — builtin annotation hashes unchanged -- [ ] `TestGoldenCanonical` passes — orcapod type hashes differ from pre-fix golden -- [ ] `TestTypeObjectHandlerWithRegistry` passes — includes module-relocation stability test -- [ ] `TestFunctionSignatureExtractorWithRegistry` passes — params and returns use same canonical form -- [ ] `TestCanonicalAnnotationStrWithRegistry` passes — registry-aware helper works for all annotation forms -- [ ] Full test suite passes - -Fixes ITL-638 -EOF -)" -``` - ---- - -## Self-Review Checklist - -**Spec coverage:** -- ✅ Locate where hashing serializes types → Tasks 3–5 (all three sites) -- ✅ Replace path-based identity with canonical name → TypeObjectHandler (Task 4) + FunctionSignatureExtractor (Task 5) -- ✅ Canonical source of truth → `LogicalTypeRegistry.logical_type_name` threaded via injection -- ✅ Consistent treatment of param and return annotations → Task 5 uses same `canonical_annotation_str` for both -- ✅ Regression tests for module-relocation scenario → `test_simulated_relocation_stable` in Tasks 4 and 5 -- ✅ Golden-value capture + diff assertion → Tasks 1 and 7 -- ✅ Backward-compat decision documented → PR body; one-time bust accepted - -**Placeholder scan:** None found. - -**Type consistency:** -- `canonical_annotation_str(annotation, registry)` — consistent across Tasks 3, 5 -- `TypeObjectHandler(logical_type_registry=...)` — consistent across Tasks 4, 6 -- `FunctionSignatureExtractor(logical_type_registry=...)` — consistent across Tasks 5, 6 -- `register_builtin_python_type_handlers(..., logical_type_registry=...)` — Task 6 only diff --git a/superpowers/plans/2026-09-03-itl-639-schema-hashing.md b/superpowers/plans/2026-09-03-itl-639-schema-hashing.md deleted file mode 100644 index 3ba9b7868..000000000 --- a/superpowers/plans/2026-09-03-itl-639-schema-hashing.md +++ /dev/null @@ -1,442 +0,0 @@ -# ITL-639 Schema Hashing Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `SchemaHandler` the explicit, canonical path for schema hashing by fixing the dispatch order in `hash_object` and implementing the handler — preserving all existing hash values with no cache bust. - -**Architecture:** Two changes in the hashing layer: (1) move handler dispatch before `_is_structure` in `semantic_hasher.py` so `Schema` objects reach `SchemaHandler` instead of being intercepted as a plain `Mapping`; (2) replace the `NotImplementedError` stub in `SchemaHandler` with a real implementation that mirrors the accidental `_expand_mapping` path, adds Arrow-translatability validation, and receives `type_converter` at registration time. - -**Tech Stack:** Python 3.10+, pytest, orcapod's `SemanticAwarePythonHasher` / `PythonTypeHandlerRegistry`, `TypeConverterProtocol`. - ---- - -## Files Changed - -| File | Action | What changes | -|------|--------|-------------| -| `src/orcapod/hashing/semantic_hashing/semantic_hasher.py` | Modify lines 172–190 | Swap order: handler dispatch moves before `_is_structure` check | -| `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | Modify lines 186–194, 529 | Replace `SchemaHandler` stub; wire `type_converter` at registration | -| `tests/test_hashing/test_type_annotation_golden.py` | Modify (append new class) | Add `TestSchemaHashBehavior` with 4 new tests | - ---- - -## Task 1: Create the feature branch - -- [ ] **Step 1: Create and check out the branch** - - ```bash - cd /home/kurouto/kurouto-jobs/d96f41f0-a0ff-442f-8ab4-1981e391b07d/orcapod-python - git checkout -b eywalker/itl-639-schema-hashing-must-go-through-arrow-representation-not - git branch --show-current - ``` - - Expected output: `eywalker/itl-639-schema-hashing-must-go-through-arrow-representation-not` - ---- - -## Task 2: Write the four new failing tests (TDD first) - -**File:** `tests/test_hashing/test_type_annotation_golden.py` - -- [ ] **Step 1: Append the new test class at the end of the file** - - Add the following at the very end of `tests/test_hashing/test_type_annotation_golden.py` (after line 274, after the `TestSchemaHashStability` class): - - ```python - # --------------------------------------------------------------------------- - # ITL-639 behaviour tests - # --------------------------------------------------------------------------- - - - class TestSchemaHashBehavior: - """Explicit behavioural tests for SchemaHandler routing and validation. - - Verifies that after ITL-639: - - Schema objects are dispatched to SchemaHandler (not _expand_mapping). - - Schema hash is stable across module renames of orcapod logical types. - - Unregistered types raise TypeError with a clear diagnostic message. - - Schema.optional_fields is intentionally excluded from the hash. - """ - - def test_schema_routes_to_schema_handler(self, hasher, monkeypatch): - """SchemaHandler.handle must be called when hashing a Schema.""" - from orcapod.hashing.semantic_hashing.builtin_handlers import SchemaHandler - - called = [] - original_handle = SchemaHandler.handle - - def patched(self, obj, h): - called.append(obj) - return original_handle(self, obj, h) - - monkeypatch.setattr(SchemaHandler, "handle", patched) - hasher.hash_object(Schema({"x": int})) - assert len(called) == 1, ( - "SchemaHandler.handle was not called — Schema is still being routed " - "through _expand_mapping instead of the registered SchemaHandler." - ) - - def test_schema_hash_stable_across_module_rename(self, hasher): - """Schema hash must not change when op.File.__module__ is mutated.""" - schema = Schema({"f": op.File}) - before = hasher.hash_object(schema).to_string() - original_module = op.File.__module__ - try: - op.File.__module__ = "orcapod.extension_types.file_type" # simulate old path - after = hasher.hash_object(schema).to_string() - finally: - op.File.__module__ = original_module - assert before == after, ( - "Schema hash changed on module rename — TypeObjectHandler is not using " - "logical_type_name for this type." - ) - - def test_schema_handler_rejects_unregistered_type(self, hasher): - """Hashing a schema with an unregistered type must raise TypeError.""" - class _Unregistered: - pass - - schema = Schema({"x": _Unregistered}) - with pytest.raises(TypeError, match="not Arrow-translatable"): - hasher.hash_object(schema) - - def test_schema_hash_ignores_optional_fields(self, hasher): - """Two schemas that differ only in optional_fields must hash identically.""" - s1 = Schema({"f": op.File}) - s2 = Schema({"f": op.File}, optional_fields={"f"}) - assert hasher.hash_object(s1).to_string() == hasher.hash_object(s2).to_string(), ( - "Schema hashes differ based on optional_fields — " - "optionality must not be part of the schema hash." - ) - ``` - -- [ ] **Step 2: Run the new tests to confirm the expected failures** - - ```bash - cd /home/kurouto/kurouto-jobs/d96f41f0-a0ff-442f-8ab4-1981e391b07d/orcapod-python - uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestSchemaHashBehavior -v - ``` - - Expected result: - - `test_schema_routes_to_schema_handler` → **FAIL** (`called` is empty — handler not reached) - - `test_schema_hash_stable_across_module_rename` → **PASS** (ITL-638 already stable) - - `test_schema_handler_rejects_unregistered_type` → **FAIL** (no TypeError raised, unregistered type falls through to `module.qualname`) - - `test_schema_hash_ignores_optional_fields` → **PASS** (`_expand_mapping` already ignores `optional_fields`) - ---- - -## Task 3: Fix dispatch order in `semantic_hasher.py` - -**File:** `src/orcapod/hashing/semantic_hashing/semantic_hasher.py`, lines 172–190. - -**What to change:** Move the handler dispatch block (currently lines 179–190) to run **before** the `_is_structure` check (currently lines 172–177). The block that starts `# Structures: expand into a tagged tree` must come **after** the handler dispatch. - -- [ ] **Step 1: Replace the middle section of `hash_object`** - - Locate this exact block in `semantic_hasher.py` (lines 172–190): - - ```python - # Structures: expand into a tagged tree, then hash the tree. - if _is_structure(obj): - expanded = self._expand_structure( - obj, _visited=frozenset(), resolver=resolver - ) - return self._hash_to_content_hash(expanded) - - # Semantic hasher dispatch: handler returns a representative Python structure - # (or a ContentHash as terminal); feed the result back into hash_object so - # that returning a plain structure is equivalent to calling hash_object on it. - handler = self._registry.get_handler(obj) - if handler is not None: - logger.debug( - "hash_object: dispatching %s to handler %s", - type(obj).__name__, - type(handler).__name__, - ) - result = handler.handle(obj, self) - return self.hash_object(result, resolver=resolver) - ``` - - Replace it with: - - ```python - # Registered handler dispatch (BEFORE _is_structure — ensures SchemaHandler - # takes priority over _expand_mapping for Schema objects, which are Mappings). - handler = self._registry.get_handler(obj) - if handler is not None: - logger.debug( - "hash_object: dispatching %s to handler %s", - type(obj).__name__, - type(handler).__name__, - ) - result = handler.handle(obj, self) - return self.hash_object(result, resolver=resolver) - - # Structures: expand into a tagged tree, then hash the tree. - if _is_structure(obj): - expanded = self._expand_structure( - obj, _visited=frozenset(), resolver=resolver - ) - return self._hash_to_content_hash(expanded) - ``` - -- [ ] **Step 2: Run the two originally-failing tests to check progress** - - ```bash - uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestSchemaHashBehavior -v - ``` - - Expected result: - - `test_schema_routes_to_schema_handler` → **FAIL** with `NotImplementedError` from `SchemaHandler` (dispatch fixed, handler reached — stub still raises) - - `test_schema_handler_rejects_unregistered_type` → **FAIL** with `NotImplementedError` (same reason) - - `test_schema_hash_stable_across_module_rename` → **FAIL** with `NotImplementedError` (now dispatches to SchemaHandler stub) - - `test_schema_hash_ignores_optional_fields` → **FAIL** with `NotImplementedError` (same) - - All four failures with `NotImplementedError` confirms dispatch is now working — the stub is reached. - ---- - -## Task 4: Implement `SchemaHandler` and wire `type_converter` - -**File:** `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` - -Two sub-changes: (a) replace the stub, (b) pass `type_converter` at registration. - -- [ ] **Step 1: Replace the `SchemaHandler` stub (lines 186–194)** - - Locate this exact block: - - ```python - class SchemaHandler: - """Hasher for ``Schema`` objects.""" - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: - if not isinstance(obj, Schema): - raise TypeError( - f"SchemaHandler: expected a Schema, got {type(obj)!r}" - ) - raise NotImplementedError("SchemaHandler is not yet implemented.") - ``` - - Replace it with: - - ```python - class SchemaHandler: - """Hasher for ``Schema`` objects. - - Canonical, explicit path for schema hashing. For each field, hashes the Python - type via ``hasher.hash_object`` (which dispatches to ``TypeObjectHandler``, using - the stable ``logical_type_name`` / Arrow extension name for registered types and the - stable ``builtins.*`` / stdlib path for native types). Field names are sorted so the - hash is deterministic regardless of insertion order. - - This produces the same hash as the previous accidental path through ``_expand_mapping`` - (Schema is a Mapping), preserving all existing hash values. - - ``Schema.optional_fields`` is intentionally excluded from the hash. Two schemas with - the same field names and types but different optionality are hash-equivalent. - Optionality is a Python-level execution contract (which parameters have defaults), - not part of the structural identity used for caching or pipeline routing. - - When ``type_converter`` is provided, every field type is verified to be - Arrow-translatable before hashing. A type that cannot be converted raises - ``TypeError`` with a diagnostic message, catching unregistered types at hash time - rather than silently falling back to a potentially unstable Python module path. - - Args: - type_converter: Optional ``TypeConverterProtocol``. When provided, validates - Arrow-translatability per field. When ``None`` (e.g. in tests without a - full ``DataContext``), validation is skipped. - """ - - def __init__(self, type_converter: "TypeConverterProtocol | None" = None) -> None: - self._type_converter = type_converter - - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: - """Hash ``obj`` by hashing each field type individually and sorting by name. - - Args: - obj: A ``Schema`` instance. - hasher: The calling ``SemanticHasherProtocol`` — used to hash each field's - Python type. - - Returns: - A ``dict[str, str]`` mapping field name to the ``to_string()`` of each - field type's hash, sorted by field name for determinism. - - Raises: - TypeError: If ``obj`` is not a ``Schema``, or if ``type_converter`` is - provided and a field type is not Arrow-translatable. - """ - if not isinstance(obj, Schema): - raise TypeError( - f"SchemaHandler: expected a Schema, got {type(obj)!r}" - ) - result: dict[str, str] = {} - for field_name, python_type in obj.items(): - if self._type_converter is not None: - try: - self._type_converter.python_type_to_arrow_type(python_type) - except (TypeError, ValueError) as exc: - raise TypeError( - f"SchemaHandler: field {field_name!r} has type " - f"{python_type!r} that is not Arrow-translatable. " - f"Every type in a schema must be Arrow-convertible — " - f"register it as an orcapod logical type or use a " - f"supported native type (int, str, float, bool, bytes, " - f"datetime, date)." - ) from exc - result[field_name] = hasher.hash_object(python_type).to_string() - # Sort by field name for determinism — matches _expand_mapping's sort_keys. - return dict(sorted(result.items())) - ``` - -- [ ] **Step 2: Wire `type_converter` into `SchemaHandler` at registration (line 529)** - - Locate this line in `register_builtin_python_type_handlers`: - - ```python - registry.register(Schema, SchemaHandler()) - ``` - - Replace it with: - - ```python - registry.register(Schema, SchemaHandler(type_converter=type_converter)) - ``` - -- [ ] **Step 3: Run the new tests — all four must pass** - - ```bash - uv run pytest tests/test_hashing/test_type_annotation_golden.py::TestSchemaHashBehavior -v - ``` - - Expected: all four tests **PASS**. - ---- - -## Task 5: Verify no golden hash values changed - -- [ ] **Step 1: Run the full test_hashing suite** - - ```bash - uv run pytest tests/test_hashing/ -v - ``` - - Expected: every test **PASS**, including: - - `TestGoldenStability::test_builtin_annotation_hashes_unchanged` - - `TestGoldenStability::test_builtin_function_hashes_unchanged` - - `TestGoldenCanonical::test_orcapod_annotation_hashes_changed` - - `TestGoldenCanonical::test_orcapod_function_hashes_changed` - - `TestSchemaHashStability::test_schema_hashes_stable` ← the 8 golden schema hashes must be byte-identical - - `TestSchemaHashStability::test_all_golden_keys_covered` - - `TestSchemaHashBehavior::test_schema_routes_to_schema_handler` - - `TestSchemaHashBehavior::test_schema_hash_stable_across_module_rename` - - `TestSchemaHashBehavior::test_schema_handler_rejects_unregistered_type` - - `TestSchemaHashBehavior::test_schema_hash_ignores_optional_fields` - - If `TestSchemaHashStability::test_schema_hashes_stable` fails with mismatched hashes, the dispatch reorder or `SchemaHandler` implementation changed the hash structure. Check that `SchemaHandler.handle` returns `dict(sorted(result.items()))` where each value is `hasher.hash_object(python_type).to_string()` — this must match exactly what `_expand_mapping` produced before (a sorted dict of `{field_name: hash_token_string}`). - -- [ ] **Step 2: Run the broader test suite to catch regressions** - - ```bash - uv run pytest tests/ -x -q - ``` - - Expected: full suite passes. If any test fails, investigate before proceeding. - ---- - -## Task 6: Update `DESIGN_ISSUES.md` if applicable - -Per the CLAUDE.md convention: check `DESIGN_ISSUES.md` for any entry matching the dispatch order bug or SchemaHandler dead code. - -- [ ] **Step 1: Check for a matching entry** - - ```bash - grep -n -i "schema.*handler\|dispatch.*order\|_expand_mapping\|ITL-639" \ - /home/kurouto/kurouto-jobs/d96f41f0-a0ff-442f-8ab4-1981e391b07d/orcapod-python/DESIGN_ISSUES.md - ``` - - If a matching entry exists, update its status to `resolved` with a brief fix note. If none exists, no action needed (the issue was tracked in Linear). - ---- - -## Task 7: Commit and open a PR - -- [ ] **Step 1: Stage the three changed files** - - ```bash - git add \ - src/orcapod/hashing/semantic_hashing/semantic_hasher.py \ - src/orcapod/hashing/semantic_hashing/builtin_handlers.py \ - tests/test_hashing/test_type_annotation_golden.py - ``` - -- [ ] **Step 2: Commit** - - ```bash - git commit -m "$(cat <<'EOF' - fix(hashing): route Schema through SchemaHandler, not _expand_mapping - - Fixes the dispatch order in hash_object so SchemaHandler is reached for - Schema objects (previously intercepted by _is_structure as a Mapping). - Implements SchemaHandler with Arrow-translatability validation and wires - type_converter at registration. Adds four regression tests. No hash values - change — all golden assertions continue to pass. Closes ITL-639. - - Co-Authored-By: Claude Sonnet 4.6 - EOF - )" - ``` - -- [ ] **Step 3: Push the branch** - - ```bash - git push -u origin eywalker/itl-639-schema-hashing-must-go-through-arrow-representation-not - ``` - -- [ ] **Step 4: Open the PR** - - ```bash - gh pr create \ - --title "fix(hashing): route Schema through SchemaHandler, not _expand_mapping (ITL-639)" \ - --body "$(cat <<'EOF' - ## Summary - - - Fixes dispatch order in `SemanticAwarePythonHasher.hash_object` so registered handlers run **before** the `_is_structure` check. This makes `SchemaHandler` reachable for `Schema` objects (previously intercepted as a plain `Mapping` by `_expand_mapping`). - - Replaces the `NotImplementedError` stub in `SchemaHandler` with the canonical implementation: sorts fields by name, hashes each Python type via `TypeObjectHandler`, and validates Arrow-translatability when a `type_converter` is available. - - Wires `type_converter` into `SchemaHandler` at registration so unregistered types are caught at hash time with a clear diagnostic error. - - Adds four regression tests: dispatch routing, module-rename stability, Arrow-translatability guard, and `optional_fields` exclusion. - - **No hash values change.** All eight golden schema hashes in `schema_hash_golden.json` remain byte-identical. - - Fixes ITL-639 - - ## Test plan - - - [ ] `uv run pytest tests/test_hashing/ -v` — all tests pass, including `TestSchemaHashStability` with unchanged golden hashes - - [ ] `uv run pytest tests/ -x -q` — full suite passes - - 🤖 Generated with [Claude Code](https://claude.com/claude-code) - EOF - )" - ``` - ---- - -## Self-Review Checklist - -**Spec coverage:** -- ✅ Dispatch order fix (`semantic_hasher.py` Task 3) -- ✅ `SchemaHandler` implementation (Task 4, Step 1) -- ✅ `type_converter` wired at registration (Task 4, Step 2) -- ✅ Module-rename regression test (Task 2 → `test_schema_hash_stable_across_module_rename`) -- ✅ SchemaHandler-is-reached test (Task 2 → `test_schema_routes_to_schema_handler`) -- ✅ Arrow-translatability guard test (Task 2 → `test_schema_handler_rejects_unregistered_type`) -- ✅ `optional_fields` excluded test (Task 2 → `test_schema_hash_ignores_optional_fields`) -- ✅ Existing golden hash stability verified (Task 5) -- ✅ Convention doc: captured in `SchemaHandler` docstring and call-site comment - -**Type consistency:** `SchemaHandler.handle` returns `dict[str, str]`, matching what `_expand_mapping` returned — no downstream type mismatch. - -**No placeholders:** All steps contain complete, runnable code. diff --git a/superpowers/plans/2026-11-09-docs-gap-report.md b/superpowers/plans/2026-11-09-docs-gap-report.md deleted file mode 100644 index 626188295..000000000 --- a/superpowers/plans/2026-11-09-docs-gap-report.md +++ /dev/null @@ -1,300 +0,0 @@ -# Orcapod Docs: Coverage Gap Report (PLT-4816, part 1) - -Date: 2026-11-09. Audited against `main` at `61d0f18e`. - -**Bottom line:** the docs now tell the truth about what they cover, but they cover only the -in-memory half of Orcapod. A newcomer can follow them through sources, function pods and -operators; for running a pipeline against a database there is now one correct section, and for -file types, side effects and troubleshooting there is nothing. The two existing hands-on -materials outside `docs/` (the tutorial notebook and the example script) do not run on `main`. -A workshop tutorial has to be written; an outline is at the end. - -## What was audited, and how - -- Every fenced `python` block in `README.md`, `docs/index.md`, `docs/getting-started.md`, - `docs/concepts/*.md` and `docs/reference/*.md` was executed against `main`, and what each - block prints was compared with the output the page shows. This is now a permanent test: - `tests/test_docs/test_doc_samples.py`. -- Before the fixes: 45 blocks, 21 passing, 24 failing (20 raised, 4 showed wrong output), - across 6 of the 8 pages that had samples. -- After the fixes: 55 blocks on 9 pages; 53 run and pass, 2 are skipped (both need a - Postgres server). The 10 additional blocks are new samples and fragments made runnable. -- `mkdocs build --strict` failed with 11 warnings and now passes; it is also a test - (`tests/test_docs/test_mkdocs_build.py`). -- Prose claims about behaviour were checked by running code, not by reading it. Where this - report says the library does something, that was observed. - -Not audited: `docs/reference/hashing.md` beyond running its one sample (487 lines of hash -derivations; it needs a reviewer who owns the hashing design), the auto-generated API pages -beyond "every name resolves and the build is clean", and the internal directories -(`docs/design`, `docs/plans`, `docs/specs`, `docs/metamorphic`). - -## Coverage along the newcomer's path - -| Step | State | Where | -|---|---|---| -| Install | Thin | `docs/index.md` | -| First pipeline | Covered well | `docs/getting-started.md` | -| Sources | Covered for in-memory sources; file/DB/polling sources missing | `docs/concepts/sources.md` | -| Function pods | Was wrong, now fixed | `docs/concepts/function-pods.md` | -| Operators (Join, Batch, GroupBy, ...) | Covered well; small gaps fixed | `docs/concepts/operators.md` | -| Running a `PipelineJob` with a database | Was wrong, now fixed; still only one section | `docs/concepts/function-pods.md` | -| Caching and re-runs | Was wrong, now fixed | `function-pods.md`, `identity.md` | -| Inspecting results | Partial | `streams.md`, `function-pods.md` | -| File and path types | Missing | (only `file-hash-caching.md`, which assumes you already use them) | -| Common errors | Missing as a page; a few now noted inline | — | - -### Install — thin - -`docs/index.md` shows `pip install orcapod` and `uv add orcapod`, nothing else. Fixed here: -the page also showed an install-from-GitHub command and `mkdocs.yml` had a `repo_url`, both -pointing at a stale location. The repository is private and Orcapod is distributed through -the organization's internal package index, so the git install, the PyPI link and the site's -repository link were removed. (Releases up to 0.1.4 are on public PyPI; the page does not -point readers there.) The section now lists the supported Python -versions (3.11, 3.12 and 3.13) and marks the exact install command as pending. - -Missing: the exact install command and index URL, the optional extras (`redis`, `ray`, -`postgresql`, `spiraldb`), and a "check your install" snippet. Two things a workshop will hit -on day one: - -- `pygraphviz` is a declared dependency and needs the Graphviz C headers to build. On a - machine without them `uv sync` fails. (Being removed under ITL-134.) -- Importing Orcapod prints nine `UserWarning: Protobuf gencode version 5.29.5 is exactly one - major version older than the runtime version 6.31.1` lines to stderr, and - `DeltaTableDatabase` writes print `WARN deltalake_core::writer::stats Skipping column ...` - lines. Harmless, but a newcomer will ask. - -The index URL and install command are not known yet; fill them in once the internal registry -is set up (ITL-810 owns the version and install strings). - -### First pipeline — covered well - -`docs/getting-started.md` walks source → inspect → function pod → inspect result, and all 11 -samples run. Fixed: the shown schema output was out of date in two samples -(`Schema({'experiment': })` is now printed as `Schema({'experiment': str})`). -Added: the exact-column rule, that every parameter needs a type (an annotation or an -`input_schema` entry), and a pointer to `PipelineJob` for keeping results. - -### Sources — partly covered - -Covered well: `DictSource`, tag versus data columns, provenance columns, `ColumnConfig`. - -Was wrong, now fixed: - -- System tag columns are named `_tag_source_id::` and `_tag_record_id::`. The - pages gave the prefix as `_tag::`, which matches no column. (`CLAUDE.md` and `.zed/rules` - repeat the old prefix in their "Column naming conventions" table; not changed here.) -- The `ColumnConfig` sample used a `source` that was not defined yet. -- `DataFrameSource` accepts Polars as well as Pandas frames; `ListSource` requires `name`. -- A `DerivedSource` comes from `job.nodes[label].as_source()`, not from a `FunctionNode`. - -Missing: a worked sample for each of `CSVSource`, `DeltaTableSource`, `DataFrameSource`, -`ListSource`; the database sources (`SQLiteTableSource`, `PostgreSQLTableSource`, -`SpiralDBTableSource`); `PollingSource` (data arriving over time, cursors); `CachedSource` and -`source.cached(db)`; what `source_id` means and when to set it. That last one matters more -than it looks — see "Things a newcomer will trip on". - -### Function pods — was wrong, now fixed - -- **Extra columns (ITL-807).** The page said a pod extracts the columns it needs from a wider - stream. It does not: an extra data column, a missing column for a parameter without a - default, or a column of an incompatible type raises `ValueError` (a parameter with a default - may be left out; the default is used). The page now states the strict rule and shows the - `SelectDataColumns` step. - Whether to relax the behaviour is the open v0.3 decision. -- **Multiple input streams.** The sample used undefined names; it is now runnable. -- Every parameter needs a type, from an annotation or the decorator's `input_schema` - (`ValueError` at decoration time otherwise); this was not stated. - -Missing entirely: - -- `FunctionPod.from_fn(...)` and context injection (`ctx_arg_name`, `InvocationContext`). -- Side effects: `side_effect_pod`, `sink_pod`, `tap_pod`, `SideEffectPod`. -- `PodConfig` versus `NodeConfig`, `job.apply_node_config(...)`. -- Ephemeral outputs (`set_ephemeral_store`). -- Executors (`LocalExecutor`, `RayExecutor`), post-run hooks, error policy - (`fail_fast` versus continue). -- Returning several outputs, optional outputs, and what happens when a function raises (in - memory, the exception propagates unchanged). - -### Operators — covered well - -Join, Batch and GroupBy were already documented with runnable samples and correct output. -Fixed: `MapTags`/`MapData` take `name_map=`, not `mapping=`; the column-selection and -`PolarsFilter` samples ran against the wrong `source` (they reused the GroupBy one, which has -no `age` or `weight` column); the "Using operators" samples used undefined streams. Added -directly, because each was one sample or one paragraph: runnable `MergeJoin`, `SemiJoin` and -`MapData` samples, the `Batch` partial-batch rule, a `Pick`/`Index` paragraph, and a section -on the fluent stream methods (`source.join(...)`, `.polars_filter(...)`, ...). - -Missing: worked samples for `Pick` and `Index`; how system tags evolve through joins and -aggregations in user terms; when to pick `Batch` versus `GroupBy` with a realistic example -followed by a function pod that consumes the lists. - -### Running a `PipelineJob` with a database — was wrong, now fixed, still thin - -The page's database example constructed `FunctionNode(function_pod=..., input_stream=..., -pipeline_database=db, result_database=db)` and called `node.run()`. `FunctionNode` no longer -takes databases and has no `run()`. The section is replaced with a tested `PipelineJob` -example (build in a `with` block, `job.run()`, read `job.nodes[label]`). - -Missing — this is the largest gap, and it is the core of the product: - -- A page of its own for `Pipeline` and `PipelineJob`: blueprint versus job, - `PipelineJob.from_pipeline(pipeline, sources=..., store=...)`, `job.bind(...)`, unbound - sources and partial runs, labels and how nodes are named. -- `DeltaTableDatabase` on disk: where files go, how `name` scopes them, what is safe to - delete. -- `job.save()` / `PipelineJob.load()` and what a loaded job can do (inspect, not run). -- Sync versus async orchestrators, observers (`StatusObserver`, `LoggingObserver`), captured - logs. -- The `orcapod` CLI. - -### Caching and re-runs — was wrong, now fixed - -Three claims in `identity.md` did not match the code: - -- "If you modify a function (change its code, ... bump its version) ... the pipeline hash - changes." Observed: editing the body, or bumping `v0.0` to `v0.1`, changes nothing and the - old stored result is returned. Only the name, the output schema and the **major** version - are part of a function's identity. -- "Phase 1: read all existing records from the shared table and yield them." Observed: a - node's stream holds the results for its current inputs only; `get_all_records()` returns - the whole shared table. -- "For in-memory sources the content hash includes the data values." True only when - `source_id` is left to default. - -All three are corrected, each with a sample that asserts the behaviour. - -Missing: how to clear or invalidate a cache deliberately (`clear_cache`, deleting a table), -`table_scope` (`pipeline_hash` versus `content_hash`), `is_stale` / `updated_since`. - -### Inspecting results — partial - -Covered: `output_schema()`, `keys()`, `iter_data()`, `as_table()`, `ColumnConfig`. Added: -`as_pandas_df()` / `as_polars_df()` / `as_lazy_frame()`, `job.nodes[label]`, -`get_all_records()`. - -Missing: reading the meta columns (`__record_id`, `__input_data_hash`, ...), tracing a value -back to its source row through `_source_*` and the system tags, `show_graph()`, the -observability reader. - -### File and path types — missing - -Observed to work: `op.File("a.txt")` and `pathlib.Path` values in a `DictSource` (schema -shows `File` / `Path`), and function pods annotated `-> Path`. None of this is documented. -`docs/concepts/file-hash-caching.md` explains caching of file hashes in detail but assumes the -reader already knows `op.File` and `op.Directory`. It was also not in the site navigation -(now added) and every sample on it failed, because it called `op.enable_file_hash_caching`, -which does not exist at the top level (see the library findings). - -Missing: a page on `op.File`, `op.Directory`, `op.Path`, `op.UPath`: what is hashed (content -versus location), how a changed file is detected, directory exclusion patterns, list-valued -file columns after `Batch`/`GroupBy`. - -### Common errors — missing - -There is no troubleshooting page. Observed messages worth collecting on one: - -| What the user did | What they get | -|---|---| -| Stream has a data column the function does not take | `ValueError: Incoming data data type Schema({...}) is not compatible with expected input schema Schema({...})` | -| Parameter without a type annotation | `ValueError: Parameter 'age' has no type annotation and is not specified in input_types.` | -| `tag_columns` names a missing column | `ValueError: tag_columns not found in table: {'zzz'}. Available columns: ['a']` | -| Every column is a tag | `ValueError: No data columns found in the table. At least one data column is required.` | -| `Join` on streams sharing a data column | `InputValidationError: Input streams are not compatible: Datas should not have overlapping keys ...` (use `MergeJoin` or rename) | -| `job.run()` without a store | `ValueError: PipelineJob.run() requires a store. Call job.bind(store=db) before run(). ...` | -| Several `output_keys` but the function returns a single value | `ValueError: Values returned by function must be sequence-like if multiple output keys are specified` | -| Edited a function, results did not change | No error. Stale cached results; bump the major version. | - -## Things a newcomer will trip on - -These are behaviours, not doc errors. They are documented truthfully now, but each will cost -workshop time unless the tutorial addresses it head-on. - -1. **Editing a function does not recompute.** By far the most likely surprise in a hands-on - session, where people edit and re-run constantly. -2. **Function pods reject extra columns** (ITL-807). Nearly every realistic source is wider - than one function's inputs, so `select_data_columns` is needed almost immediately. -3. **An explicit `source_id` is the source's identity.** Two `DictSource`s with the same - `source_id` and schema but different rows have the same `content_hash()`. -4. **Default versions differ.** `@function_pod` defaults to `version="v0.0"`; - `FunctionPod.from_fn` defaults to `"v1.0"`. -5. **`drop_unmapped` defaults differ.** `MapData(...)` keeps unmapped columns; - `stream.map_data(...)` drops them. - -## Suspected library problems found - -Reported to the coordinator rather than fixed or worked around; the docs describe what the -code does today. - -1. `orcapod.enable_file_hash_caching` does not exist. The page (and the function's own - docstring, which speaks of `op.File`) reads as if a top-level export was intended. - Reproduction: `import orcapod as op; op.enable_file_hash_caching()` → - `AttributeError: module 'orcapod' has no attribute 'enable_file_hash_caching'`. Works as - `from orcapod.contexts import enable_file_hash_caching`. -2. Empty data gives a misleading error. `DictSource(data=[], tag_columns=["a"])` → - `ValueError: tag_columns not found in table: {'a'}. Available columns: []`. `CLAUDE.md` - says this case raises `ValueError("Table is empty")`. -3. A schema mismatch writes a stray line to stderr before raising: - `Key 'age' not found in parameter types.` (or `Type mismatch for key 'age': ...`). - Reproduction: apply a pod taking `(weight, height)` to a stream that also has `age`. -4. The mismatch error text reads "Incoming data data type" (a doubled word). -5. `examples/save_and_load_pipelines.py` fails on `main`: - `AttributeError: 'DeltaTableDatabase' object has no attribute 'register_tracker'` - (it calls `Pipeline("my_pipeline", database)`; `Pipeline` no longer takes a database). -6. `notebooks/tutorials/01_introduction_to_orcapod.ipynb` uses the removed API - (`Pipeline(name=..., pipeline_database=...)`, 19 uses of `FunctionNode(`). Not executed - here, but it cannot run as written. -7. Items 4 and 5 under "Things a newcomer will trip on" (inconsistent defaults) look - unintended. -8. Stale docstrings, fixed in this PR with the coordinator's approval because they broke the - strict docs build: `DeltaTableDatabase.add_records` documented `arrow_table` and - `id_columns`; `get_records_by_ids` documented `add_record_id_column` and - `preserve_input_order`. Neither function has those parameters. - -Also stale but outside this PR's ownership: the "Lower-level details on pipeline workflow -(tentative)" section of `README.md` (it describes `pipeline.run(execution_engine=...)`; one -plainly wrong line, the `auto_compile` default, was corrected), and the README has no user-facing -introduction at all. - -## Proposed outline: self-paced workshop tutorial - -Audience: scientists comfortable with Python, new to Orcapod. One running dataset throughout -(for example, a small set of recording sessions: a CSV of subjects, a folder of per-session -files). Each part is a page under `docs/tutorial/`, about 20 minutes, ends with two or three -exercises with solutions, and every sample runs under the docs test harness. - -0. **Setup (10 min).** Install, extras, check the install, the warnings you can ignore. -1. **Your first stream.** `DictSource` and `CSVSource`; tags versus data; `output_schema`, - `iter_data`, `as_pandas_df`. *Exercise:* choose tag columns for a given table; provoke and - read the two construction errors. -2. **Computing with function pods.** `@function_pod`, type annotations, single and multiple - outputs, the exact-column rule and `select_data_columns`. *Exercise:* fix a pod that fails - on a wide stream. -3. **Reshaping with operators.** `join`, `semi_join`, `map_data`, `polars_filter`; `Join` - versus `MergeJoin`. *Exercise:* combine subjects with measurements and filter. -4. **Many rows into one.** `GroupBy` and `Batch`, then a function pod over `list[T]` inputs - (per-subject summary). *Exercise:* per-session mean from per-probe rows. -5. **Keeping results: `PipelineJob`.** Record in a `with` block, `job.run()`, `job.nodes`, - `InMemoryArrowDatabase` versus `DeltaTableDatabase`. *Exercise:* run, quit Python, run - again, show nothing was recomputed. -6. **Re-runs and caching.** Add rows and re-run; edit a function and see the stale result; - bump the major version. What identity means (`content_hash` / `pipeline_hash`) in one - page, linking to the concept page. *Exercise:* predict which of four edits recompute. -7. **Working with files.** `op.File`, `op.Directory`, `Path` outputs; pods that read and - write files; file hash caching for large files. *Exercise:* a pod that writes one output - file per session. -8. **Inspecting and tracing.** Meta and provenance columns, `get_all_records()`, - `as_source()` to start a second pipeline from stored results, `show_graph()`. -9. **When things go wrong.** The common-errors table above as a troubleshooting page, plus - error policy and captured logs. -10. **Where next (optional).** Side-effect pods (`sink_pod`), context injection, executors - and Ray, `PollingSource`, saving and loading jobs. - -Prerequisites to write it: the `PipelineJob` concept page and the file-types page listed above -(parts 5 and 7 need them), and a decision on ITL-807, since part 2 is written around the -strict rule. Replace or delete `notebooks/tutorials/01_...ipynb` and -`examples/save_and_load_pipelines.py` at the same time, so that no hands-on material on `main` -fails. diff --git a/superpowers/plans/2026-11-09-itl-134-remove-pygraphviz.md b/superpowers/plans/2026-11-09-itl-134-remove-pygraphviz.md deleted file mode 100644 index 792a78fbe..000000000 --- a/superpowers/plans/2026-11-09-itl-134-remove-pygraphviz.md +++ /dev/null @@ -1,87 +0,0 @@ -# Remove pygraphviz and Restore Green CI (ITL-134) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Drop the unused `pygraphviz` dependency so `uv sync` no longer compiles a C extension against Graphviz headers, which is what turned every CI job red when `ubuntu-latest` moved to ubuntu-26.04. - -**Architecture:** No source change. `pygraphviz` leaves `pyproject.toml` and `uv.lock`; every workflow stops installing `libgraphviz-dev` but keeps the `graphviz` apt package, because `src/orcapod/pipeline/graph.py` uses the pure-Python `graphviz` package, which shells out to the `dot` binary. The MinIO job in `tests.yml` is a separate failure handled in Task 4 after the coordinator's decision. - -**Tech Stack:** uv, GitHub Actions, pytest. - ---- - -## Evidence gathered before planning - -- `grep -rniE "pygraphviz|nx_agraph|to_agraph|graphviz_layout|agraph|nx_pydot|pydot"` over the repo (excluding `.git`, `.venv`, `uv.lock`) matches only `pyproject.toml`, the workflow apt lines (`libgraphviz-dev`), and historical `superpowers/` plans. Nothing in `src/`, `tests/`, `test-objective/`, `notebooks/`, `examples/`, `bench/`, `design/`, `docs/`, or `README.md`. -- networkx imports pygraphviz only inside `networkx.drawing.nx_agraph`; no `nx.drawing` / `nx_agraph` / `nx_pydot` reference exists in `src/`, `tests/`, or `test-objective/`. -- `src/orcapod/pipeline/graph.py:613,669` import `graphviz` (pure Python, needs `dot` at render time only). -- Local repro of the CI failure class: `uv sync` on a machine without Graphviz headers fails with `Failed to build pygraphviz==1.14 ... 'graphviz/cgraph.h' file not found`. -- CI: failing jobs run on `Image: ubuntu-26.04`; error is `graphviz_wrap.c:5480: passing argument 5 of 'gvRenderData' from incompatible pointer type`. -- README and docs contain no Graphviz header install instructions today. -- MinIO: Docker Hub `minio/minio` returns 404; `quay.io/minio/minio` returns 401 to anonymous pulls. `ghcr.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z` exists (digest `sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9`). - -## Files - -- Create: `tests/test_packaging.py` (regression guard) -- Modify: `pyproject.toml` (remove one dependency line), `uv.lock` (regenerated) -- Modify: `.github/workflows/{run-tests,run-objective-tests,run-postgres-tests,_license-check,release}.yml`; delete `.github/workflows/tests.yml` -- Modify: `README.md`, `docs/index.md` (one sentence each on optional `dot`) -- Modify: `CHANGELOG.md` (one entry under `[Unreleased]` → `### Changed`) -- Modify: `DESIGN_ISSUES.md` (one entry appended at the end) - -### Task 1: Regression test - -- [ ] **Step 1: Write the failing test** in `tests/test_packaging.py`: - -```python -"""Packaging guards for the dependency list in ``pyproject.toml``.""" - -from __future__ import annotations - -import tomllib -from pathlib import Path - -PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml" - - -def test_pygraphviz_is_not_a_dependency(): - """``pygraphviz`` needs Graphviz C headers to build and nothing imports it (ITL-134).""" - project = tomllib.loads(PYPROJECT.read_text())["project"] - declared = list(project["dependencies"]) - for extra in project.get("optional-dependencies", {}).values(): - declared.extend(extra) - assert not [d for d in declared if d.lower().startswith("pygraphviz")] -``` - -- [ ] **Step 2: Show it fails.** The project venv cannot be built before the fix, so run it outside the project: - `uv run --no-project --with pytest pytest -c /dev/null --rootdir . --noconftest tests/test_packaging.py` - Expected: FAIL, listing `pygraphviz>=1.14`. - -### Task 2: Remove the dependency - -- [ ] **Step 1:** Delete the line ` "pygraphviz>=1.14",` from `[project] dependencies` in `pyproject.toml`. -- [ ] **Step 2:** `uv lock`, then `git diff --stat uv.lock` and `git diff uv.lock`. Expected: only the three pygraphviz hunks removed (dependency entry, `requires-dist` entry, `[[package]]` block). If any other package version moved, restore `uv.lock` from git, delete those three hunks by hand, and confirm with `uv lock --check`. -- [ ] **Step 3:** `uv sync --locked --all-extras --dev` succeeds locally with no Graphviz headers installed. -- [ ] **Step 4:** `uv run pytest tests/test_packaging.py` passes. - -### Task 3: CI, docs, changelog - -- [ ] **Step 1:** In every workflow replace `apt-get install -y graphviz libgraphviz-dev` with `apt-get install -y graphviz` (`tests.yml` is removed in Task 4). Keep `runs-on: ubuntu-latest`; pin to `ubuntu-24.04` only if the PR's CI shows another image-related failure. -- [ ] **Step 2:** README "Running Tests Locally" and `docs/index.md` "Installation": add one sentence that the Graphviz `dot` binary is optional and only needed to render pipeline graphs; no development headers are required. -- [ ] **Step 3:** CHANGELOG `[Unreleased]` → `### Changed`: add a "Removed `pygraphviz` dependency (ITL-134)" entry. -- [ ] **Step 4:** Append a resolved entry to `DESIGN_ISSUES.md`. -- [ ] **Step 5:** `uv run pytest tests/` (full suite) and the license check command from `_license-check.yml`. Record failures and skips by name. -- [ ] **Step 6:** Commit `fix(deps): remove unused pygraphviz dependency (ITL-134)`, push, open the PR against `main` with `Fixes ITL-134`. - -### Task 4: MinIO job (`tests.yml`) - -Coordinator decision: remove the workflow in this PR. The official image no longer exists on -Docker Hub or quay.io, a third-party image is the developer's call, and the workflow is -otherwise redundant with `run-tests.yml`. S3 integration tests currently skip everywhere -(the testcontainers fallback uses the same missing image); restoring them is tracked as ITL-819. - -- [ ] `git rm .github/workflows/tests.yml`. Do not touch `tests/test_databases/conftest.py`. - -### Task 5: Verify on CI - -- [ ] Wait for the PR's checks; report the actual conclusions of `test (3.11)`, `test (3.12)`, `objective-test`, `license-check`, and the MinIO job. diff --git a/superpowers/plans/2026-11-09-itl-810-v0.2-release-prep.md b/superpowers/plans/2026-11-09-itl-810-v0.2-release-prep.md deleted file mode 100644 index 07b29b495..000000000 --- a/superpowers/plans/2026-11-09-itl-810-v0.2-release-prep.md +++ /dev/null @@ -1,167 +0,0 @@ -# Orcapod Python v0.2 Release Prep (ITL-810) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Prepare everything needed for the developer to publish `orcapod` 0.2.0 through the -GitHub Actions Release workflow, without triggering it. - -**Architecture:** Documentation- and metadata-only change. No library behaviour is edited. -The release is cut from `main` by `.github/workflows/release.yml`; the version comes from the -git tag via `hatch-vcs`, so nothing in the tree carries a version number to bump. - -**Tech Stack:** uv, hatchling + hatch-vcs, pytest, pip-licenses, GitHub Actions, Linear. - -**Branch:** `eywalker/itl-810-cut-orcapod-python-v02-by-november-1` from `origin/main` (`61d0f18e`). -**PR body:** `Part of ITL-810` (no `Fixes` / `Closes`). - ---- - -## Findings that shape the plan (established before writing it) - -1. **Patch parity — nothing to port.** `git diff --stat v0.1.2 v0.1.4` touches exactly one - file, `src/orcapod/core/executors/ray.py`, in `RayExecutor._get_remote_fn`. `v0.1.4` - builds the remote as `ray.remote(**opts)(wrapper)` when `opts` is truthy and - `ray.remote(wrapper)` otherwise, inside the cache-miss branch. `main` has the same logic - as a conditional expression (from `e5efd39b` #233 and `4f7f6fb9` #235): - `git diff v0.1.4 origin/main -- src/orcapod/core/executors/ray.py` shows only that - syntactic difference. `main` also has regression tests for the empty-opts path in - `tests/test_core/test_regression_fixes.py` (the `fake_remote(fn=None, **opts)` tests). - `v0.1.3` (`dcd54575`) is an intermediate state fully superseded by `v0.1.4` (`16002bc0`). -2. **`v0.1.4`'s commit is on no branch.** `origin/patch` points at `dcd54575` (`v0.1.3`); - `16002bc0` is reachable only from the tag. Report, do not fix. -3. **`0.1.3` and `0.1.4` were published outside the Release workflow.** PyPI has them; - TestPyPI stops at `0.1.2`; GitHub Releases stop at `v0.1.2`; no Release workflow run - exists for them. -4. **The Release workflow has not succeeded since 2026-07-11** (`v0.1.2`). The one later - attempt (2026-08-20) failed in `Test (Python 3.11)` and was fixed by #254 the same day; - #254 and #256 then re-pinned every Action. The workflow is untested end to end since. -5. **`CHANGELOG.md` has only an `[Unreleased]` section**, last edited 2026-06-29, whose - content shipped in `v0.1.1`. Nothing records `v0.1.2`–`v0.1.4`. -6. **Two of the breaking changes the brief names shipped in `v0.1.2`**, not in the 29 - commits since: the `NodeConfig` → `PodConfig` / `NodeConfig` split (#221, ITL-512) and - the SpikeInterface split (#220, ITL-473). They were never changelogged, so they are - recorded in the 0.2.0 migration section, labelled as having shipped in 0.1.2. -7. **`pygraphviz` blocks a local `uv sync`** (needs Graphviz C headers, not installed here). - Coordinator decision: do not install Graphviz; ITL-134 removes the dependency before - the cut. Run pre-flight with `--no-install-package pygraphviz` and `uv run --no-sync`; - the unmodified clean install must be re-run after ITL-134 merges. -8. **`Run Tests` is red on `main`** on every commit since at least 2026-08-26, solely - because the `spiral-integration` job gets `403 Permission table:view denied` from - SpiralDB. The `test (3.11)` / `test (3.12)` jobs pass. Not part of the Release pre-flight. - -## File structure - -| File | Action | Responsibility | -|---|---|---| -| `CHANGELOG.md` | Modify | New `## [0.2.0]` entry; relabel the stale `[Unreleased]` block as `[0.1.1]`; add short `[0.1.2]`–`[0.1.4]` notes | -| `README.md` | Modify | Add Installation section; correct the Releasing paragraph; fix typo | -| `pyproject.toml` | Modify | Trove classifiers for Python 3.11 / 3.12; `Changelog` and `Repository` URLs. No dependency changes | -| `RELEASING.md` | Modify | Pre-release checklist pointer, recovery from a failed run, lineage note on `v0.1.3` / `v0.1.4` | -| `docs/index.md` | Modify (one line) | Fix the stale `walkerlab` GitHub URL in the install snippet | -| `mkdocs.yml` | Modify (one line) | Fix the stale `walkerlab` `repo_url` | -| `superpowers/plans/2026-11-09-v0.2-release-checklist.md` | Create | Ordered publish steps + ITL-810 checklist evidence | - -No source or test files change. No `DESIGN_ISSUES.md` entry is needed (no bug fixed). - ---- - -### Task 1: Patch-parity evidence - -- [ ] **Step 1:** Run and record: - -```bash -git diff --stat v0.1.2 v0.1.4 -git diff v0.1.4 origin/main -- src/orcapod/core/executors/ray.py -git branch -a --contains 16002bc0 -``` - -Expected: one file changed; a diff that only reformats the `if opts:` branch into a -conditional expression; no branch listed. - -- [ ] **Step 2:** After the environment exists (Task 4), run the covering tests: - -```bash -uv run --no-sync pytest tests/test_core/test_regression_fixes.py -q -k "remote" -``` - -Expected: PASS. No port, no code change. - -### Task 2: CHANGELOG - -- [ ] **Step 1:** Rename `## [Unreleased]` to `## [0.1.1] — 2026-06-29` (its content is an - ancestor of the `v0.1.1` tag: `git merge-base --is-ancestor a8d47051 v0.1.1`). -- [ ] **Step 2:** Insert above it, newest first: `## [0.2.0] — unreleased`, then - `## [0.1.4] — 2026-07-17`, `## [0.1.3] — 2026-07-17`, `## [0.1.2] — 2026-07-10`. -- [ ] **Step 3:** The 0.2.0 entry has these sections in this order: summary paragraph; - `### Breaking changes and migration`; `### Added`; `### Changed`; `### Fixed`; - `### Known limitations`. Every item cites its PR number and Linear id from - `git log --oneline v0.1.2..origin/main`. In-flight work gets a placeholder line (the - capitalized marker followed by `(pending merge):`) for ITL-816, ITL-813, ITL-817, - ITL-604 (#248) and ITL-134. All were resolved by 2026-11-10. -- [ ] **Step 4:** Verify coverage — every PR number in the 29 commits appears in the entry: - -```bash -for n in $(git log --format=%s v0.1.2..origin/main | grep -o '(#[0-9]*)$' | tr -d '(#)'); do - grep -q "#$n" CHANGELOG.md || echo "MISSING #$n"; done -``` - -Expected: no output. - -- [ ] **Step 5:** Commit: `docs(changelog): add v0.2.0 entry and backfill 0.1.x headings (ITL-810)` - -### Task 3: README, pyproject metadata, docs strings, RELEASING - -- [ ] **Step 1:** `README.md` — add an Installation section (pip / uv, Python >= 3.11 tested - on 3.11 and 3.12, extras `redis`, `ray`, `postgresql`, `spiraldb`, `all`, SpikeInterface - extension pointer); replace "tag a commit on `main`" with the workflow-triggered wording - used by `RELEASING.md`; fix "reproducbile". -- [ ] **Step 2:** `pyproject.toml` — add classifiers - `Programming Language :: Python :: 3.11`, `Programming Language :: Python :: 3.12`, and - `Repository` / `Changelog` entries under `[project.urls]`. Do not touch any dependency. -- [ ] **Step 3:** `docs/index.md:19` and `mkdocs.yml:3` — `walkerlab` → `nauticalab`. -- [ ] **Step 4:** `RELEASING.md` — add "Before you cut", "If a run fails part-way", and a - note that `v0.1.3` / `v0.1.4` are not ancestors of `main`. -- [ ] **Step 5:** Confirm the lockfile is still current: `uv lock --check`. Expected: exit 0. -- [ ] **Step 6:** Commit: `docs: align README, metadata and release docs with 0.2.0 (ITL-810)` - -### Task 4: Release checks - -- [ ] **Step 1:** Tests, per Python version, in an isolated environment: - -```bash -uv sync --locked --all-extras --dev --no-install-package pygraphviz --python 3.11 -uv run --no-sync --python 3.11 pytest -m "not postgres" --tb=short -q -rs -``` - -and the same for 3.12. Record passed / failed / skipped counts and skip reasons by name. - -- [ ] **Step 2:** License check (as `_license-check.yml`, default extras only): - -```bash -uv sync --locked --dev --no-install-package pygraphviz -uv run --no-sync pip-licenses --allow-only="MIT;Apache-2.0;Apache 2.0;Apache Software License;BSD-2-Clause;BSD-3-Clause;BSD License;BSD;ISC;LGPL-3.0-only;MPL-2.0;Mozilla Public License;Python-2.0;PSF-2.0;Python Software Foundation License;Unlicense" --partial-match -``` - -- [ ] **Step 3:** Build: `uv build` into a scratch output directory. Inspect wheel metadata - (`Requires-Python`, `Requires-Dist`, `Provides-Extra`, classifiers) and file list. -- [ ] **Step 4:** Smoke test: create a clean venv per Python version, install the built wheel - with `pygraphviz` excluded, then `import orcapod`, print - `importlib.metadata.version("orcapod")` (there is no `orcapod.__version__`), run a - minimal source → function pod pipeline, and run `orcapod --help`. -- [ ] **Step 5:** Full suite as the standing rules require: `uv run --no-sync pytest tests/`. - -Fix only contained packaging / metadata problems; report everything else. - -### Task 5: Release checklist document - -- [ ] **Step 1:** Write `superpowers/plans/2026-11-09-v0.2-release-checklist.md` with: merge - order of the open PRs, post-merge re-verification (including the unmodified clean - install after ITL-134), workflow inputs, post-release verification, who to notify, and - the five ITL-810 checklist items with their evidence. -- [ ] **Step 2:** Commit: `docs(release): add v0.2 release checklist (ITL-810)` - -### Task 6: Finish - -- [ ] Rebase onto current `origin/main`; re-run the suite if `main` moved. -- [ ] Push, open PR against `main` (`Part of ITL-810`), `orca linear attach`, - `orca worktree set --worktree current --pr `, send `worker_done`. diff --git a/superpowers/plans/2026-11-09-itl-813-mergejoin-empty-input.md b/superpowers/plans/2026-11-09-itl-813-mergejoin-empty-input.md deleted file mode 100644 index 47520af41..000000000 --- a/superpowers/plans/2026-11-09-itl-813-mergejoin-empty-input.md +++ /dev/null @@ -1,186 +0,0 @@ -# ITL-813 MergeJoin Empty-Input Fix Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `MergeJoin` with a zero-row input returns a zero-row table with the predicted joined schema instead of raising, and both `Join` and `MergeJoin` gain empty-input regression tests. - -**Architecture:** Two null-typed-on-zero-rows constructs in `MergeJoin.binary_static_process` are given explicit Arrow types. Non-empty behaviour (Arrow types, hashes, stored schemas) is left byte-identical: explicit types are applied only where inference has no rows to infer from. No public API change. - -**Tech Stack:** Python 3.12, PyArrow, Polars, pytest, `uv run --no-sync` (pygraphviz cannot build on this machine; see coordinator note). - ---- - -## Root cause (verified on main 61d0f18e by probe, before any code change) - -`pa.array([])` has Arrow type `null`. `MergeJoin.binary_static_process` hits that twice: - -1. **Helper key** (`merge_join.py` lines 222, 225): `pa.array([0] * len(table))` is `null` for a - zero-row side and `int64` for the other. Polars raises - `SchemaError: datatypes of join keys don't match - _common: i64 on left does not match _common: null on right`. - Reproduced for left-empty, right-empty, colliding and non-colliding. -2. **Merged list columns** (lines 299, 301, 308): `pa.array(merged_vals)` and - `pa.array(merged_sources)` are `null` when the join yields zero rows. With both sides empty - the helper keys agree (both `null`), the join succeeds, and the failure moves to schema - reconstruction: `ValueError: A null type field may not be non-nullable`. Once (1) is fixed - this path is reached for every empty-side join with a colliding column. - -`Join` already uses `type=pa.int8()` (lines 137, 155) and passes; it has no tests for it. - -## Probe results for the other operators (zero-row input) - -| Operator | Zero-row result | -|---|---| -| Join | pass (already typed) | -| MergeJoin | **fails** (two constructs above) — fixed here | -| SemiJoin | pass (no helper key; Polars semi join on real tag keys) | -| Batch (size 0, size 2, drop_partial) | pass: 0 rows, no null-typed columns | -| GroupBy | pass: 0 rows, no null-typed columns | -| Select/Drop Tag/Data columns | pass | -| MapTags / MapData | pass | -| PolarsFilter (empty in; all rows dropped) | pass | -| Pick / Index | pass | - -Not zero-row bugs, found by the probe, **reported only** (they reproduce on non-empty input): - -- `PolarsFilter` output loses `nullable=False` (Polars round trip), so the materialized schema - is `int | None` where `output_schema()` predicts `int`. -- `Batch`, `GroupBy`, `MergeJoin` predict `list[T | None]` for a nullable element column but - materialize `list[T]`. -- `MergeJoin` materializes system-tag columns as non-nullable (null-count inference) when the - inputs' system-tag fields are nullable (e.g. after a `PolarsFilter`), while the prediction - keeps them nullable. The MergeJoin system-tag test therefore compares system-tag column - names, not full schemas. -- `Pick` / `Index` reject a nullable `dict` / `list` column (`dict[str, int] | None` "is not a - supported pick target"). - -## File structure - -- Modify: `src/orcapod/core/operators/merge_join.py` — type the helper key; add module-level - helper `_list_array` and use it for merged data/source columns. -- Create: `tests/test_core/operators/test_empty_inputs.py` — all zero-row operator tests. -- Modify: `DESIGN_ISSUES.md` — one entry appended at the end. - ---- - -### Task 1: Failing tests - -**Files:** -- Create: `tests/test_core/operators/test_empty_inputs.py` - -- [ ] **Step 1: Write the tests.** Helpers: - - `_stream(columns, tags, nullable=True)` builds an `ArrowTableStream` from a `pa.table`, - optionally casting every field to `nullable=False`. - - `_filtered_empty(columns, tags)` builds an `ArrowTableSource` with rows and runs - `PolarsFilter(predicates=[pl.lit(False)]).static_process(...)` — the real fede shape: an - empty stream that still carries system-tag and source-info columns. - - `_assert_empty_result(op, streams, expected_columns)`: - `op.validate_inputs(*streams)`; `result = op.static_process(*streams)`; for - `cfg in ({}, {"system_tags": True})`: table has 0 rows, no field is `pa.null()`, - and `result.output_schema(columns=cfg) == op.output_schema(*streams, columns=cfg)` - (materialized stream vs prediction — never `process()`, which is circular). Default-config - column names equal `expected_columns` as a set. Finally `len(op.process(*streams).as_table()) == 0`. - - Test classes: - - `TestJoinEmptyInput`: issue snippet; two streams with empty at index 0 / 1 / both; three - streams with empty at index 0 / 1 / 2 / all; a filtered-empty (system-tag-carrying) case. - - `TestMergeJoinEmptyInput`: issue-correction snippet; parametrized - `side in (left, right, both)` × `colliding in (False, True)`; the colliding case uses - non-nullable columns, and asserts the data schema type is `list[int]` and the Arrow type - is a list; a filtered-empty (system-tag-carrying) non-colliding case; a colliding case - with source-info columns requested (`columns={"source": True}` has no null-typed field). - - `TestOtherOperatorsEmptyInput`: parametrized over SemiJoin (3 shapes), Batch (3 configs), - GroupBy, the four column-selection operators, MapTags, MapData, PolarsFilter (empty in, - and all rows dropped), Pick, Index. Batch/GroupBy use non-nullable inputs and - PolarsFilter nullable inputs, because of the reported non-zero-row nullability - mismatches above (stated in the test docstring). - -- [ ] **Step 2: Run, verify MergeJoin fails and everything else passes.** - -Run: `uv run --no-sync pytest tests/test_core/operators/test_empty_inputs.py -q` -Expected: every `TestMergeJoinEmptyInput` test FAILS with `SchemaError` (`_common`) or -`ValueError: A null type field may not be non-nullable`; Join and other-operator tests PASS. - -- [ ] **Step 3: Commit** `test(operators): add zero-row input tests for all operators (ITL-813)` - -### Task 2: Fix MergeJoin - -**Files:** -- Modify: `src/orcapod/core/operators/merge_join.py` - -- [ ] **Step 1: Type the helper key** (both sites), same as `Join`: - -```python - # Use int8 explicitly so a zero-row table doesn't produce a null-typed - # column that Polars rejects as a join key (same as Join). - left_table = left_table.add_column( - 0, COMMON_JOIN_KEY, pa.array([0] * len(left_table), type=pa.int8()) - ) - right_table = right_table.add_column( - 0, COMMON_JOIN_KEY, pa.array([0] * len(right_table), type=pa.int8()) - ) -``` - -- [ ] **Step 2: Add the module-level helper** after the imports: - -```python -def _list_array(values: list[Any], element_type: "pa.DataType | None") -> "pa.Array": - """Build the Arrow array for a merged (list-valued) column. - - With rows present the type is inferred from the values, exactly as before. - With zero rows inference yields a ``null``-typed array, which cannot be a - non-nullable field and does not match the predicted ``list[T]`` schema, so - the list type is built from ``element_type`` instead. - - Args: - values: One list per output row. - element_type: Arrow type of the column's elements before merging, or - ``None`` if unknown. - - Returns: - The list array for the merged column. - """ - if values or element_type is None: - return pa.array(values) - if isinstance(element_type, pa.ExtensionType): - # Arrow cannot embed an extension type in a list value field. - element_type = element_type.storage_type - return pa.array(values, type=pa.list_(element_type)) -``` - -- [ ] **Step 3: Use it** at the three untyped sites. Lines 299 and 301 become - `merged_array = _list_array(merged_vals, elem_arrow_type)`. For the source column capture - the element type before the column is dropped: - -```python - if has_source: - source_idx = joined.column_names.index(left_source_col) - source_elem_type = joined.schema.field(left_source_col).type - joined = joined.drop(left_source_col) - joined = joined.drop(right_source_col) - source_array = _list_array(merged_sources, source_elem_type) - joined = joined.add_column(source_idx, left_source_col, source_array) -``` - -- [ ] **Step 4: Run** `uv run --no-sync pytest tests/test_core/operators/ -q` — all PASS. - -- [ ] **Step 5: Commit** `fix(merge_join): type helper key and merged columns for zero-row inputs (ITL-813)` - -### Task 3: Log, verify, ship - -- [ ] **Step 1:** Append one `DESIGN_ISSUES.md` entry (status `resolved`, with **Fix:** note). -- [ ] **Step 2:** `uv run --no-sync pytest tests/ -q` — full suite; record pre-existing - failures and skips by name. -- [ ] **Step 3:** Commit `docs(design-issues): log MergeJoin zero-row fix (ITL-813)` together - with this plan. -- [ ] **Step 4:** Rebase on `origin/main` if it moved, rerun the suite, push, open the PR - against `main` (`Fixes ITL-813`), `orca linear attach`, `orca worktree set --pr`, send - `worker_done`. - -## Self-review - -- Spec coverage: helper-key fix (T2.1); Join tests two/three streams, one/all empty (T1); - MergeJoin left/right/both incl. colliding column (T1); schema == predicted (T1 helper); - other operators checked (probe table) and pinned by tests (T1); same-kind fix for merged - columns (T2.2–2.3); reportable findings listed above. -- No change to public API, documented behaviour, hash values, or stored schemas for - non-empty inputs — no coordinator approval needed. diff --git a/superpowers/plans/2026-11-09-itl-816-iter-data-chunk-layouts.md b/superpowers/plans/2026-11-09-itl-816-iter-data-chunk-layouts.md deleted file mode 100644 index b217b3e22..000000000 --- a/superpowers/plans/2026-11-09-itl-816-iter-data-chunk-layouts.md +++ /dev/null @@ -1,153 +0,0 @@ -# ITL-816 `ArrowTableStream.iter_data` Chunk-Layout Fix Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `ArrowTableStream.iter_data` yields every row, correctly paired, whatever the chunk -layout of the tag columns, data columns and source-info table. - -**Architecture:** `iter_data` currently calls `to_batches()` separately on the tag-column -selection and the data-column selection of `self._table`, then `zip`s the two batch lists. -`Table.to_batches()` cuts at the union of the chunk boundaries of the *selected* columns, so -the two lists differ whenever tag and data columns are chunked differently (which is exactly -what the orchestrator's `hstack_tables(concat(tags), concat(data))` produces). `zip` -truncates (rows dropped) and unequal batch lengths mis-pair rows. A second bug of the same -family sits two lines below: `self._source_info_table.slice(i, 1)` uses the *batch-local* -index `i`, so every row after the first batch gets another row's source info. - -The fix iterates batches of the **whole** `self._table` once (`self._table.to_batches()` -yields batches in which all columns are aligned by construction), selects tag and data columns -out of each batch, and indexes source info by a global row offset from a single -`to_pylist()`. No per-row `table.slice`, no copy of the table. - -**Tech Stack:** Python, PyArrow, pytest (`uv run pytest`). - -**No change** to public API, schemas, hashes or record ids: the same single-row `RecordBatch` -slices reach `Tag` / `Data` as before, for the rows that were previously yielded correctly. - ---- - -## File structure - -- Modify: `src/orcapod/core/streams/arrow_table_stream.py` (`iter_data` only; drop the unused - `repeat` import). -- Create: `tests/test_core/streams/test_iter_data_chunk_layouts.py` (regression tests). -- Modify: `DESIGN_ISSUES.md` (one entry appended at the end). - -### Task 1: Failing regression tests - -**Files:** -- Create: `tests/test_core/streams/test_iter_data_chunk_layouts.py` - -- [ ] **Step 1: Write the tests** - -Helper `_chunked_stream(tag_sizes, data_sizes, source_sizes=None)` builds a table whose tag -columns (`id`, `grp`) are `pa.chunked_array`s cut at `tag_sizes`, whose data columns (`x`, -`y`) are cut at `data_sizes`, and optionally a per-row `_source_x` column cut at -`source_sizes`. Row `i` has `id=i`, `grp=f"g{i}"`, `x=i*10`, `y=f"v{i}"`, -`_source_x=f"src::{i}"`, so any mis-pairing is detectable. - -Parametrized layouts (tag sizes, data sizes), all summing to the same row count: - -| id | tag | data | -|---|---|---| -| tag-single/data-many | `[6]` | `[1,1,1,1,1,1]` | -| tag-many/data-single | `[2,2,2]` | `[6]` | -| unequal-boundaries | `[4,2]` | `[1,5]` | -| fewer-tag-chunks | `[3,3]` | `[2,2,2]` | -| fewer-data-chunks | `[1,2,3]` | `[5,1]` | -| reporter-shape (6 rows → 4) | `[4,2]` | `[2,2,1,1]` | -| empty-chunk | `[0,6]` | `[3,0,3]` | -| zero-length data chunks (cache-hit shape) | `[1]*6` | `[1,0,0,1,0,1,0,0,1,1,0,1]` | -| zero-length tag chunks | `[1,0,0,1,0,1,0,0,1,1,0,1]` | `[1]*6` | -| zero-length chunks both sides | `[0,1,0,2,0,3]` | `[3,0,0,2,1,0]` | -| same-layout (control) | `[2,4]` | `[2,4]` | - -Assertions per layout: -1. `len(list(stream.iter_data())) == stream.as_table().num_rows == n`. -2. For each yielded `(tag, data)`: `data["x"] == tag["id"] * 10`, `data["y"] == - f"v{tag['id']}"`, `tag["grp"] == f"g{tag['id']}"`, and ids come out in order `0..n-1`. -3. `as_table(columns={"content_hash": True})` succeeds (it appends one hash per iterated row). -4. A second `iter_data()` call (cache path) returns the same pairs. - -Plus: a no-tag-column stream with multi-chunk data yields all rows; a per-row source-info test -(`_source_x` chunked `[2,2,2]`, others single chunk and vice versa) asserting -`data.source_info()["x"] == f"src::{id}"`; an empty (0-row) stream yields nothing. - -- [ ] **Step 2: Run to verify failure** - -Run: `uv run pytest tests/test_core/streams/test_iter_data_chunk_layouts.py -v` -Expected: the mismatched-layout cases FAIL (fewer rows than `as_table()` or mis-paired -values); the control passes. - -### Task 2: Fix `iter_data` - -**Files:** -- Modify: `src/orcapod/core/streams/arrow_table_stream.py` - -- [ ] **Step 1: Replace the body of the uncached branch** - -```python - if self._cached_elements is None: - cached_elements = [] - tag_columns = list(self._all_tag_columns) - data_columns = list(self._data_columns) - empty_tag = Tag({}) if not tag_columns else None - - # Indexed by absolute row position; the source-info table has its - # own chunk layout, so it must never be indexed batch-locally. - source_info_rows = self._source_info_table.to_pylist() - - # Batch the whole table once. ``Table.to_batches`` cuts at the - # union of the chunk boundaries of every column, so tag and data - # columns of a batch always cover the same rows. Batching the tag - # and data selections separately and zipping them drops or - # mis-pairs rows whenever the two are chunked differently (ITL-816). - row_offset = 0 - for batch in self._table.to_batches(): - tag_batch = batch.select(tag_columns) if tag_columns else None - data_batch = batch.select(data_columns) - for i in range(batch.num_rows): - if tag_batch is not None: - tag = Tag( - tag_batch.slice(i, 1), # type: ignore - data_context=self.data_context, - ) - else: - tag = cast(Tag, empty_tag) - - data = Data( - data_batch.slice(i, 1), - source_info=source_info_rows[row_offset + i], - data_context=self.data_context, - ) - - yield tag, data - - cached_elements.append((tag, data)) - row_offset += batch.num_rows - self._cached_elements = cached_elements -``` - -Remove `from itertools import repeat`. - -- [ ] **Step 2: Run the new tests** — `uv run pytest tests/test_core/streams -v` → all PASS. - -- [ ] **Step 3: Measure** — time `sum(1 for _ in stream.iter_data())` on 100k-row tables - (single chunk, aligned 1k chunks, with a per-row source column), interleaving the pre-fix - class (loaded from `git show origin/main:src/orcapod/core/streams/arrow_table_stream.py` - into a scratch module) and the fixed class in one process. The fix must not be slower. - -- [ ] **Step 4: Commit** — `fix(streams): iterate ArrowTableStream rows over aligned batches (ITL-816)` - -### Task 3: Audit - -Check every site in `src/orcapod` that calls `to_batches()`, iterates `.chunks`, zips two -row sequences, or slices rows by index; classify each as safe / fixed / out-of-ownership. -Record the list in the PR body and `worker_done`. Sites in the orchestrator materialization -code, Join system-tag handling and `side_effects.py` belong to ITL-817: report, do not edit. - -### Task 4: Log, verify, ship - -- [ ] Append one `DESIGN_ISSUES.md` entry (status `resolved`, **Fix:** note). -- [ ] `uv run pytest tests/` — full suite green; list pre-existing failures/skips by name. -- [ ] Rebase on `origin/main` if it moved, push, open PR against `main` with `Fixes ITL-816`. diff --git a/superpowers/plans/2026-11-09-itl-817-system-tag-run-stability.md b/superpowers/plans/2026-11-09-itl-817-system-tag-run-stability.md deleted file mode 100644 index 32d36cc6f..000000000 --- a/superpowers/plans/2026-11-09-itl-817-system-tag-run-stability.md +++ /dev/null @@ -1,108 +0,0 @@ -# ITL-817 System-Tag Run Stability Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make tag column order deterministic through `Join` / `MergeJoin` and row materialization, pin run-stable completion tracking with regression tests, and document the positional default record id — without changing any stored identity. - -**Architecture:** Investigation (below) showed system tags stay row-aligned through `Join`, the `FunctionJobNode` cache and the orchestrator. Two things were found instead: (1) the reported signature reproduces only when a source has a run-stable `source_id`, no `record_id_column`, and a varying row order, because the default record id is positional; (2) `Join` and `MergeJoin` build `tag_columns` from a `set`, so tag column order depends on `PYTHONHASHSEED`, which makes `pa.concat_tables` raise when a buffer mixes cached and fresh rows. This plan fixes (2), tests and documents (1), and writes up changing the default as a proposal. - -**Tech Stack:** Python, PyArrow, Polars, deltalake, pytest, `uv run`. - -**Coordinator decision (2026-11-09):** Option A approved. Zero stored identities may change. Option B (content-derived default record ids) is a proposal only. PR body says `Part of ITL-817`. - ---- - -## Investigation record - -Measured on `61d0f18e`, one process per run, one on-disk `DeltaTableDatabase`. - -| Configuration | Result | -|---|---| -| 2 sources → `Join` (cross product) → function pod → `sink_pod`, same input order, 6 processes with different `PYTHONHASHSEED` | 9/0/0/0/0/0 | -| Same, shared tag key | 9/0/0 | -| Same, explicit `source_id` + `record_id_column`, rows shuffled each run | 9/0/0/0, system tags identical per key | -| Same, default `source_id` (table hash), rows shuffled | every id changes (source id itself changes) | -| Same, explicit `source_id`, **no** `record_id_column`, rows shuffled, ITL-816 neutralised with `combine_chunks` | **9/9/9, record ids move between rows, user tags and data correct — the reported signature** | -| 2 function pods, growing inputs (cached + fresh mix) | `ArrowInvalid: Schema at index N was different` in 7 of 10 runs (tag column order) | -| Same as previous, one run | tag chunks `[1]*6` vs data chunks `[1,1,1,1,1,0,0,0,0,0,1]` → `iter_data` yields 5 of 6 (ITL-816; result-cache hit returns `Data` with zero-length chunks) | - -Code paths: - -- Positional record id: `src/orcapod/core/sources/stream_builder.py` `_make_provenance_token` returns `row_{i}` when `record_id_column` is `None`; `_make_record_id` hashes `source_id::token`. -- Set-ordered tags: `src/orcapod/core/operators/join.py` `static_process` (`tag_keys` is a `set`, `tag_columns=tuple(tag_keys)`); `src/orcapod/core/operators/merge_join.py` `binary_static_process` (`all_tag_keys` is a `set`). -- Cached rows use compile-time order: `FunctionJobNode._load_cached_entries` takes `self._input_stream.keys()[0]`. -- Concatenation that fails: `SyncPipelineOrchestrator._materialize_as_stream`, `StaticOutputOperatorPod._materialize_to_stream`. -- `arrow_hasher.hash_table` ignores column order (verified), so tag order never changes a record id. - -## File structure - -- Modify `src/orcapod/core/operators/join.py` — tag order from input streams in call order. -- Modify `src/orcapod/core/operators/merge_join.py` (only the block after `sort_system_tag_values`; stay clear of lines ~220-226). -- Modify `src/orcapod/utils/arrow_utils.py` — add `concat_tables_by_name`. -- Modify `src/orcapod/pipeline/sync_orchestrator.py`, `src/orcapod/core/operators/static_output_pod.py` — use it. -- Modify `src/orcapod/core/sources/stream_builder.py`, `arrow_table_source.py` — docstrings. -- Create `tests/test_core/operators/test_join_tag_order.py`, `tests/test_pipeline/test_system_tag_run_stability.py`. -- Create `superpowers/specs/2026-11-09-itl-817-content-derived-record-ids.md`. -- Append one entry to `DESIGN_ISSUES.md`. - ---- - -### Task 1: Deterministic tag order in Join and MergeJoin - -**Files:** `tests/test_core/operators/test_join_tag_order.py`, `join.py`, `merge_join.py` - -- [ ] **Step 1: failing test.** A subprocess script builds two `ArrowTableSource`s (`probe`/`x`, `method`/`w`), applies `Join()` (and `MergeJoin()` with a colliding data column) and prints `json.dumps(list(next(iter(stream.iter_data()))[0].keys()))`. The test runs it with `PYTHONHASHSEED` in `1..6` via `subprocess.run([sys.executable, "-c", script], env=...)` and asserts every output equals `["probe", "method"]`. -- [ ] **Step 2:** `uv run --no-sync pytest tests/test_core/operators/test_join_tag_order.py -v` → FAIL (seeds 1-3 give `["method", "probe"]`). -- [ ] **Step 3: implement.** In `Join.static_process` build - -```python -tag_order: list[str] = [] -for s in streams: - for key in s.keys()[0]: - if key not in tag_order: - tag_order.append(key) -``` - - and pass `tag_columns=tuple(tag_order)`. This is the order `output_schema` already predicts (`union_schemas` over the streams in call order). In `MergeJoin.binary_static_process` pass `tag_columns=tuple(left_tag_keys) + tuple(k for k in right_tag_keys if k not in left_tag_keys)`. -- [ ] **Step 4:** rerun → PASS. Run `tests/test_core/operators`. -- [ ] **Step 5:** commit `fix(operators): deterministic tag column order in Join and MergeJoin (ITL-817)`. - -### Task 2: Materialization tolerates per-row column order - -**Files:** `arrow_utils.py`, `sync_orchestrator.py`, `static_output_pod.py`, `tests/test_pipeline/test_system_tag_run_stability.py` - -- [ ] **Step 1: failing tests.** Build two rows whose tags are `Tag({"a": 1, "b": 2})` and `Tag({"b": 3, "a": 4})`; call `StaticOutputOperatorPod._materialize_to_stream(rows)` and `SyncPipelineOrchestrator._materialize_as_stream(rows, node_stub)`; assert two rows come back with `a`/`b` values still paired. -- [ ] **Step 2:** run → FAIL with `ArrowInvalid: Schema at index 1 was different`. -- [ ] **Step 3: implement** `arrow_utils.concat_tables_by_name(tables)`: reorder every table to the first table's column order with `table.select(names)` when the name sets are equal, then `pa.concat_tables`; if the sets differ, leave the tables alone so Arrow raises as before. Use it for both tag and data concatenations in both materializers. -- [ ] **Step 4:** rerun → PASS. -- [ ] **Step 5:** commit `fix(pipeline): align per-row tables by column name when materializing buffers (ITL-817)`. - -### Task 3: End-to-end run-stability tests - -**Files:** `tests/test_pipeline/test_system_tag_run_stability.py` - -- [ ] **Step 1:** module-level `scale` function pod; helper `run_once(db_path, probes, methods, shared_key)` that builds a fresh `PipelineJob(name="p", store=DeltaTableDatabase(db_path))` with two `ArrowTableSource`s (`source_id=`, `record_id_column=`), `Join`, `scale.pod`, and a `sink_pod` that records deliveries, and returns `(delivered, {user_tag_key: system_tags})` captured by wrapping `side_effects._execute_side_effect_row` with `monkeypatch`. -- [ ] **Step 2:** tests, parametrized over cross-product and shared-key joins: (a) same order twice → second run delivers 0; (b) rows shuffled between three runs → 9/0/0 and identical system tags per key; (c) growing inputs 1×2 → 2×3 → 3×3 → 3×3 delivers exactly the new rows each time and never changes an existing key's system tags (cached + fresh mix). -- [ ] **Step 3:** if (c) trips ITL-816 (`iter_data` truncation, owned by the ITL-816 worker), neutralise it in a fixture that wraps `ArrowTableStream.__init__` with `combine_chunks()` and say so in the fixture docstring; do not touch `arrow_table_stream.py`. -- [ ] **Step 4:** run → PASS; confirm (c) fails on the pre-fix commit (`git stash`-free: check out `HEAD~2 -- src` into a scratch worktree or revert the two fix commits temporarily). -- [ ] **Step 5:** commit `test(pipeline): pin run-stable sink completion under reorder and mixed cache (ITL-817)`. - -### Task 4: Documentation, proposal, design log - -- [ ] Docstrings: `stream_builder._make_provenance_token`, `SourceStreamBuilder.build`, `ArrowTableSource` — state that without `record_id_column` the record id is positional, so a reordered row gets a new record id and is treated as a new downstream entry (recorded and delivered again); pass `record_id_column` for order-independent identity. -- [ ] If `docs/` has a user-facing sources page mentioning `record_id_column`, add one paragraph only. -- [ ] Write `superpowers/specs/2026-11-09-itl-817-content-derived-record-ids.md` (blast radius, duplicate-row rule, migration option). -- [ ] Append one `DESIGN_ISSUES.md` entry for ITL-817. -- [ ] Mirror nothing in `CLAUDE.md` / `.zed/rules` unless an instruction changes (none does). -- [ ] commit `docs(sources): document positional default record ids; propose content-derived ids (ITL-817)`. - -### Task 5: Verify and ship - -- [ ] `uv run --no-sync pytest tests/` full suite; record failures and skips by name. -- [ ] Rebase on `origin/main` if it moved; rerun. -- [ ] Push, open PR against `main` with `Part of ITL-817`, attach to Linear, set worktree PR, send `worker_done`. - -## Self-review - -- Spec coverage: hash-seed regression test (Task 1), MergeJoin and other set-built tag orders checked (Task 1; `SemiJoin` already uses schema order), reorder + mixed-cache pins (Task 3), docs (Task 4), proposal (Task 4), no identity change (hasher is column-order-insensitive; no value, name or hash is touched). -- No placeholders; names used consistently (`concat_tables_by_name`). diff --git a/superpowers/plans/2026-11-09-itl-825-python-313-canonical-type-names.md b/superpowers/plans/2026-11-09-itl-825-python-313-canonical-type-names.md deleted file mode 100644 index 6d28cf6d5..000000000 --- a/superpowers/plans/2026-11-09-itl-825-python-313-canonical-type-names.md +++ /dev/null @@ -1,132 +0,0 @@ -# ITL-825 Python 3.13 Canonical Type Names Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Canonical type names and every hash derived from them are identical on Python 3.11, 3.12 and 3.13, unchanged on 3.11/3.12, and the repo officially supports 3.13. - -**Architecture:** One dependency-free helper module (`orcapod/utils/canonical_type_names.py`) resolves a stdlib type's *defining* module to the *public* module it is exported from, when the defining module is a private submodule (`pathlib._local` → `pathlib`). Every identity-string site that prints a type routes through it. A frozen set of stdlib private submodules that were already the defining module on 3.11/3.12 is exempt, so no string changes on those versions. - -**Tech Stack:** Python 3.11–3.13, uv, pytest, GitHub Actions. - ---- - -## Evidence (established before planning, on main @ 07e5a2ea) - -Stdlib survey (all importable stdlib modules, 3.11 / 3.12 / 3.13, plus 3.14 for a forward look): - -| Type | 3.11 | 3.12 | 3.13 | -|---|---|---|---| -| `pathlib.{Path,PosixPath,WindowsPath,PurePath,PurePosixPath,PureWindowsPath}` | `pathlib.X` | `pathlib.X` | `pathlib._local.X` | -| `zipfile.Path`, `zipfile.CompleteDirs` | `zipfile.X` | `zipfile._path.X` | `zipfile._path.X` | -| `re.error` (qualname rename) | `re.error` | `re.error` | `re.PatternError` | -| `signal.ItimerError` (qualname rename) | `signal.itimer_error` | `signal.itimer_error` | `signal.ItimerError` | - -Everything else users plausibly annotate with (`datetime.*`, `decimal.Decimal`, `uuid.UUID`, -`fractions.Fraction`, `ipaddress.*`, `re.Pattern`/`re.Match`, `enum.*`, `collections.abc.*`, -typing constructs) has the same module, qualname and `inspect.formatannotation` output on all three. - -Stdlib private submodules whose classes a public ancestor re-exports: - -- 3.11 ∪ 3.12 (frozen exemption list — strings must not change): `concurrent.futures._base`, - `ctypes._endian`, `importlib.metadata._collections`, `importlib.metadata._meta`, - `xml.sax._exceptions`, `zipfile._path`, `zoneinfo._common`, `zoneinfo._tzpath`. -- New in 3.13: `pathlib._local`, `pathlib._abc` (only `UnsupportedOperation`, new in 3.13). -- New in 3.14: `pathlib._os`, `concurrent.interpreters._queues`, `compression.zstd._zstdfile` - (all new types; the rule gives them their public names). - -Affected Orcapod code paths (probe on a default-configured context): - -- `PythonDataFunction._function_signature_hash` — `hash_utils.get_function_signature` uses - `str(param)`, `canonical_annotation_str(annotation)` **without** a type converter, and - `f"-> {ret}"`. Differs on 3.13 for any `Path` annotation. -- `PythonDataFunction._function_content_hash` — `hash_utils.get_function_components` emits - `annotations:{k}:{v}` using `str(type)`. Differs on 3.13. - Both are stored as function-variation columns of the result cache. -- `TypeObjectHandler` (`type:.`) and `canonical_annotation_str` fallback — - differ on 3.13 for *unregistered* pathlib types (`PurePath` family) and when no type - converter is supplied. `pathlib.Path` itself resolves to `orcapod.path` first. -- NOT affected: `uri`, `content_hash`, `pipeline_hash`, schema hashes, record ids of a pod - annotated with `Path` (they go through the registered logical type). - -Out of scope, reported only: `zipfile.Path` already differs between 3.11 and 3.12 (changing -either needs developer approval); the two qualname renames above are exception classes. - -## File structure - -- Create `src/orcapod/utils/canonical_type_names.py` — the resolver. -- Modify `src/orcapod/hashing/hash_utils.py` — `canonical_annotation_str`, `get_function_signature`, `get_function_components`. -- Modify `src/orcapod/utils/function_info.py` — `annotations:` component. -- Modify `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` — `TypeObjectHandler`. -- Modify `src/orcapod/hashing/semantic_hashing/semantic_hasher.py` — `_handle_unknown`. -- Create `tests/test_hashing/test_canonical_type_names.py` — pinned literals + golden hashes. -- Modify `.github/workflows/{run-tests,run-objective-tests,run-postgres-tests,release}.yml`, `pyproject.toml`, `DESIGN_ISSUES.md` (one appended entry), `CLAUDE.md` + `.zed/rules` (one implementation note). - -### Task 1: Failing tests - -- [ ] Create `tests/test_hashing/test_canonical_type_names.py` with: - - `EXPECTED_CANONICAL` — literal `canonical_annotation_str` strings (no converter) taken from - 3.12 for: the six pathlib classes, `Path | str`, `Optional[Path]`, `list[Path]`, - `dict[str, Path]`, `tuple[Path, ...]`, `datetime.{datetime,date,time,timedelta}`, - `decimal.Decimal`, `uuid.UUID`, `fractions.Fraction`, `ipaddress.IPv4Address`, - `re.Pattern`, `enum.Enum`, `collections.abc.Sequence[int]`, `collections.abc.Mapping[str, int]`, - `typing.Any`, `typing.Literal["a", "b"]`, `concurrent.futures.Future` (exempt, stays - `concurrent.futures._base.Future`), `io.StringIO` (stays `_io.StringIO`). - - user-defined and third-party types keep `__module__` exactly. - - `TypeObjectHandler` output and `semantic_hasher.hash_object(PurePath)` literal. - - golden: `get_function_signature`, `get_function_components` annotations line, and - `PythonDataFunction` `function_signature_hash` / `function_content_hash` literals for a - module-level function annotated with `Path`, `Path | str`, `list[Path]`, - `dict[str, Path]`, `datetime.datetime` (literals generated on 3.12 from main). -- [ ] Run on 3.13: `uv run --python 3.13 pytest tests/test_hashing/test_canonical_type_names.py tests/test_hashing/test_hash_utils.py tests/test_hashing/test_function_info_extractors.py -q` → FAIL with `pathlib._local`. -- [ ] Run on 3.12 → PASS except helper-import tests (helper does not exist yet). - -### Task 2: Resolver - -- [ ] Create `src/orcapod/utils/canonical_type_names.py`: - -```python -def canonical_module_name(obj: object) -> str | None: ... -def canonical_type_name(tp: type) -> str: ... # "." -def stabilize_type_names(text: str, *annotations: object) -> str: ... -``` - - Rule in `canonical_module_name`: return `__module__` unchanged unless (a) its top-level - package is in `sys.stdlib_module_names`, (b) a non-leading component starts with `_`, - (c) the module is not in `_LEGACY_PRIVATE_STDLIB_MODULES`, and (d) walking `__qualname__` - from the public ancestor (`sys.modules`) reaches the very same object. Then return the - public ancestor. - - `stabilize_type_names` walks an annotation (`__origin__`, `__args__`, nested lists) and - replaces `"<__module__>."` with the canonical name in `text` for every class - whose module was resolved. It is a no-op when nothing was resolved, which is what keeps - 3.11/3.12 output byte-identical. - -### Task 3: Route identity-string sites through it - -- [ ] `canonical_annotation_str` fallback: `stabilize_type_names(inspect.formatannotation(annotation), annotation)`. -- [ ] `get_function_signature`: stabilise each `str(param)` against its annotation and the `-> {ret}` text against the return annotation. -- [ ] `get_function_components` (both copies): stabilise each `{k}:{v}` against `v`. -- [ ] `TypeObjectHandler`: `canonical_module_name(obj) or ""`. -- [ ] `_handle_unknown`: `canonical_module_name(type(obj))`. -- [ ] Run Task 1 tests on 3.11, 3.12, 3.13 → PASS. Commit. - -### Task 4: Prove no change on 3.11/3.12 - -- [ ] Re-run the probe (`scratchpad/probe/probe.py`, 438 identity values) on the branch for all - three versions; branch-3.11 == main-3.11, branch-3.12 == main-3.12, and branch-3.13 == - branch-3.12 except time-based record ids and the pre-existing `zipfile.Path` / - `typing.NamedTuple` rows. - -### Task 5: Official 3.13 support - -- [ ] Add `"3.13"` to the matrices in the four workflows; add classifiers - `Programming Language :: Python :: 3.11|3.12|3.13` to `pyproject.toml` (no lockfile change: - classifiers are not locked; verify `uv lock --check`). -- [ ] Full suite on 3.11, 3.12, 3.13: `uv run --python X pytest tests/ -q`; also `test-objective/`. -- [ ] Fix contained 3.13-only failures; report the rest. - -### Task 6: Docs, hygiene, PR - -- [ ] Append `DESIGN_ISSUES.md` entry (resolved, ITL-825); add the implementation note to `CLAUDE.md` and `.zed/rules`. -- [ ] `ruff check` / `ruff format --check` on new files, `basedpyright` on changed files vs main. -- [ ] Rebase on origin/main, push, open PR (`Fixes ITL-825`), attach to Linear, report CI. diff --git a/superpowers/plans/2026-11-09-itl-826-org-required-ci.md b/superpowers/plans/2026-11-09-itl-826-org-required-ci.md deleted file mode 100644 index 352354fac..000000000 --- a/superpowers/plans/2026-11-09-itl-826-org-required-ci.md +++ /dev/null @@ -1,173 +0,0 @@ -# ITL-826 Org Required CI Compliance Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `main` pass the four checks of the metamorphic-labs `Python — Required CI` workflow (ruff lint, ruff format, basedpyright, pytest without Graphviz) without changing runtime behaviour. - -**Architecture:** Five reviewable commits in a fixed order: skip-guard for the `dot`-dependent tests, a formatter-only commit with the ruff config, hand-reviewed lint fixes, a basedpyright config plus committed baseline, then contributor docs and pre-commit hooks. No dependency or `uv.lock` change. - -**Tech Stack:** uv, ruff 0.15.8 and basedpyright 1.38.4 (both resolved from `uv.lock`), pytest. - ---- - -## Measured state on `origin/main` (07e5a2ea) - -All commands run through `uv run --python 3.11`. - -| Check | Result | -| -- | -- | -| `uv run --with ruff ruff check .` | 271 errors: F401 147, E402 40, E721 21, E702 19, F541 11, F841 10, E731 7, F811 4, E722 3, E741 3, F821 3, invalid-syntax 2, E712 1 | -| `uv run --with ruff ruff format --check .` | 281 of 450 files would be reformatted | -| `uv run --with basedpyright basedpyright` | 2080 errors, 31200 warnings (tests 1238, src 678, test-objective 84, notebooks 76, examples 3, superpowers 1) | -| `uv run pytest tests/` (3.11 and 3.12, no `dot`) | 3 failed, 4821 passed, 93 skipped, 2 xfailed | -| `uv run pytest test-objective/` (3.11 and 3.12) | 567 passed, 7 xfailed | - -**Tool versions.** `uv run --with ruff` does not install the latest ruff here: ruff and -basedpyright are already in the project environment (dev group / dependencies), so the -`--with` requirement is satisfied by the locked versions, ruff 0.15.8 and basedpyright -1.38.4. The CI log of run 42724396559 confirms it: it reports exactly 271 lint errors, the -0.15.8 number. The latest ruff (0.17.4 via `uvx`) reports 1670 errors with a much larger -default rule set. The lock file is therefore the version pin; this PR does not change it. - -**Line length.** Formatter diff by line length (ruff 0.15.8, target py311): - -| line-length | files reformatted | changed lines | -| -- | -- | -- | -| 79 | 356 | 18029 | -| 88 | 281 | 8636 | -| 100 | 321 | 9002 | -| 120 | 331 | 10944 | - -88 is the smallest on both measures. It is also the ruff default the existing pre-commit -hook has been applying. - -**Notebooks.** `notebooks/` holds 8 `.ipynb` files. One -(`old_tutorials/02_orcapod_basic_usage copy.ipynb`) does not parse; two would be -reformatted with JSON churn; six import `orcabridge`, a package that no longer exists, and -account for 76 basedpyright errors. `notebooks/` is excluded from ruff and basedpyright by -config. No file is deleted. - -**The three failing tests** (all in `tests/test_pipeline/test_graph_rendering.py`): -`TestRenderGraphFullPath::test_render_with_output_path`, -`TestRenderGraphFullPath::test_render_with_show_mocked`, -`TestPipelineShowGraphFullPath::test_show_graph_with_show_mocked`. - -## Rules for every task - -- Never run `uv lock` or `uv add`. Check `git status` shows no `uv.lock` change after each step. -- Do not change behaviour in `src/orcapod/hashing` (ITL-825). Lint findings there are fixed only when the fix is provably behaviour-neutral, else suppressed with a reasoned `# noqa`. -- Do not touch `CHANGELOG.md`. -- A lint fix that would change runtime behaviour, a file deletion, or an exclusion other than `notebooks/` and build artefacts goes to the coordinator through `ask` first. -- End every commit message with `Co-Authored-By: Claude Fable 5.2 `. - ---- - -### Task 0: Snapshot the public surface - -**Files:** none in the repo (scratchpad script only). - -- [ ] **Step 1:** Write `surface.py` in the scratchpad. It walks `orcapod` with `pkgutil.walk_packages`, imports every module (recording import failures by exception type), and writes `{module_name: sorted(dir(module))}` as JSON. -- [ ] **Step 2:** Run `uv run --python 3.11 python surface.py before.json` on the untouched tree. - -### Task 1: Skip the `dot`-dependent tests when Graphviz is absent - -**Files:** Modify `tests/test_pipeline/test_graph_rendering.py`. - -- [ ] **Step 1:** The failing state already exists: the three tests fail with `graphviz.backend.execute.ExecutableNotFound` when `dot` is not on `PATH`. -- [ ] **Step 2:** Add after the imports: - -```python -import shutil - -requires_dot = pytest.mark.skipif( - shutil.which("dot") is None, - reason="Graphviz `dot` executable not found on PATH; install Graphviz to run " - "the tests that render a graph to an image", -) -``` - -- [ ] **Step 3:** Decorate exactly the three tests above with `@requires_dot`. Leave the test bodies unchanged: where `dot` exists the real rendering still runs. -- [ ] **Step 4:** `uv run pytest tests/test_pipeline/test_graph_rendering.py -rs`. Expected without `dot`: 3 skipped with the reason above, 0 failed. Then run with a stub `dot` first on `PATH` to confirm the marker is evaluated from `PATH` and the three tests are selected to run rather than skipped. -- [ ] **Step 5:** Commit `test(graph_rendering): skip dot-dependent tests when Graphviz is not installed`. - -The repository's own `Run Tests` workflow installs Graphviz, so on the PR it must show these three tests as passed, not skipped. Check its log. - -### Task 2: ruff config and formatter run - -**Files:** Modify `pyproject.toml`; formatter output across the repo. - -- [ ] **Step 1:** Add to `pyproject.toml`: - -```toml -[tool.ruff] -target-version = "py311" -line-length = 88 -# Tutorial notebooks are not linted or formatted: reformatting rewrites their JSON, and -# the ones under old_tutorials/ target a package that no longer exists. -extend-exclude = ["notebooks"] -``` - -- [ ] **Step 2:** `uv run --with ruff ruff format .` -- [ ] **Step 3:** `uv run --with ruff ruff format --check .` reports every file formatted. `git diff --stat` touches only `.py` files and `pyproject.toml`. -- [ ] **Step 4:** `uv run pytest tests/ -q` and `uv run pytest test-objective/ -q` on 3.11: same counts as Task 1. -- [ ] **Step 5:** Commit `style: apply ruff format across the repository` (config plus formatter output, nothing else). - -### Task 3: Lint fixes - -**Files:** as reported by `ruff check`; no new files. - -- [ ] **Step 1:** `uv run --with ruff ruff check . --fix` (safe fixes only). Review the diff of every `__init__.py` and every file under `src/`. -- [ ] **Step 2:** Re-exports. `src/orcapod/execution_engines/__init__.py`, `nodes/__init__.py`, `operators/__init__.py`, `sources/__init__.py`, `streams/__init__.py`: keep the import and make it explicit (`from x import y as y`, or an `__all__` entry). Never remove a name from a package `__init__`. -- [ ] **Step 3:** F821 (3, all `Path` used inside a string annotation without an import, in `contexts/__init__.py` and `pipeline/job.py`): add `from pathlib import Path` under `if TYPE_CHECKING:`. No runtime effect. -- [ ] **Step 4:** F811 (4). Two are a repeated local import (`pa`, `asyncio`): remove the repeat. Two are duplicate test class names (`TestOperatorNodeAsyncExecute` in `tests/test_channels/test_node_async_execute.py`, `TestSideEffectFunctionPodDecorator` in `tests/test_core/side_effect_function/test_side_effect_function_pod.py`): the second definition shadows the first, so 4 + 2 tests are never collected. Rename the second class (`TestOperatorNodeAsyncExecuteCacheModes`, `TestSideEffectFunctionPodDecoratorForms`). Run the six newly collected tests. If any fails, stop and `ask`. -- [ ] **Step 5:** E402. Restructure where the late import has no reason (move `logger = ...` below the imports; drop the duplicated `_JSONSCHEMA_AVAILABLE` line in `contexts/registry.py` only if the two assignments are identical). Where a test must set `sys.path` or environment before importing, use `# noqa: E402` with the reason. -- [ ] **Step 6:** Remaining rules by hand: E721 (`type(x) == T` to `is`, only where the operands are types; behaviour-identical for classes), E702 (split statements), E731 (lambda to `def`), E741 (rename `l`), E712, F841 (drop the unused binding, keep the call), F541. E722 bare `except`: `except:` and `except Exception:` differ for `KeyboardInterrupt`/`SystemExit`, so a bare except is kept and suppressed with a reasoned `# noqa: E722` rather than changed. -- [ ] **Step 7:** `uv run --with ruff ruff check .` reports `All checks passed!`; `ruff format --check .` still passes. -- [ ] **Step 8:** Run `surface.py after.json` and diff against `before.json`. Expected: no name removed from any module. Record the comparison for the PR. -- [ ] **Step 9:** Full suites on 3.11 and 3.12. Expected: 4821 + 6 passed (the unshadowed tests), 96 skipped, 2 xfailed, 0 failed. -- [ ] **Step 10:** Commit `fix(lint): make ruff check pass`. - -### Task 4: basedpyright config and baseline - -**Files:** Modify `pyproject.toml`; create `.basedpyright/baseline.json`. - -- [ ] **Step 1:** Add to `pyproject.toml` (no `typeCheckingMode`, so the default `recommended` mode stays): - -```toml -[tool.basedpyright] -# Setting `exclude` replaces the built-in defaults, so they are repeated here. -exclude = [ - "**/node_modules", - "**/__pycache__", - "**/.*", - "build", - "dist", - "site", - "notebooks", -] -``` - -- [ ] **Step 2:** Compare the error set under `--pythonplatform Linux` and `Darwin`. If they differ, pin `pythonPlatform = "Linux"` and `pythonVersion = "3.11"` so the baseline written on macOS matches what CI sees; if identical, leave them unset. -- [ ] **Step 3:** Fix cheap real defects first. Known: the three undefined `Path` names (fixed in Task 3). `orcapod.hashing.legacy_core` does not exist, but the import sits in a `try/except ImportError` that defines public names as `None`; changing it alters the public surface of `orcapod.hashing`, so it is baselined and reported, not fixed. `cloudpickle` and `spiral` are optional packages that are not installed: baselined. -- [ ] **Step 4:** `uv run --python 3.11 --with basedpyright basedpyright --writebaseline`. -- [ ] **Step 5:** `uv run --with basedpyright basedpyright` exits 0 with `0 errors`. Record the baselined error count by top-level directory. -- [ ] **Step 6:** Add `x: int = "not an int"` to a source file; the same command must exit 1 with exactly 1 error. Revert the line. -- [ ] **Step 7:** If the baseline exceeds 500 kB, exempt it from the `check-added-large-files` pre-commit hook in Task 5. -- [ ] **Step 8:** Commit `chore(types): add basedpyright config and baseline of existing errors`. - -### Task 5: Contributor docs and pre-commit - -**Files:** Modify `CLAUDE.md`, `.zed/rules`, `.pre-commit-config.yaml`, `DESIGN_ISSUES.md`. - -- [ ] **Step 1:** Add a "Lint, format and type checks" section with identical text to `CLAUDE.md` and `.zed/rules`: the four commands; new and changed code must be ruff-clean, ruff-formatted and add no basedpyright errors; the baseline is regenerated only to remove fixed entries, never to hide a new error; the `dot` binary is optional and the tests that need it skip without it; tool versions come from `uv.lock`. -- [ ] **Step 2:** `.pre-commit-config.yaml`: set the `ruff-pre-commit` rev to `v0.15.8` (the locked version) and add the `ruff-check` hook ahead of `ruff-format`. -- [ ] **Step 3:** Append one entry to the end of `DESIGN_ISSUES.md` for ITL-826: the baseline as type debt, with the counts by directory. -- [ ] **Step 4:** Commit `docs: document lint, format and type-check requirements; align pre-commit with CI`. - -### Task 6: Verify and open the PR - -- [ ] **Step 1:** Rebase onto `origin/main` if it moved; rerun the suites. -- [ ] **Step 2:** Full suites on 3.11 and 3.12, `test-objective/`, and the four org commands on 3.11. -- [ ] **Step 3:** Push; open the PR against `main` with `Fixes ITL-826` in the body; `orca linear attach`; `orca worktree set --worktree current --pr `. -- [ ] **Step 4:** Wait for CI. Required green: ruff, typecheck, license-check, pytest 3.11, pytest 3.12. pytest 3.13 may fail only the three ITL-825 hashing tests; read its log and confirm. -- [ ] **Step 5:** Send `worker_done` with the acceptance list. diff --git a/superpowers/plans/2026-11-09-itl-827-packaging-cleanup.md b/superpowers/plans/2026-11-09-itl-827-packaging-cleanup.md deleted file mode 100644 index e24738939..000000000 --- a/superpowers/plans/2026-11-09-itl-827-packaging-cleanup.md +++ /dev/null @@ -1,162 +0,0 @@ -# Packaging Cleanup for v0.2 (ITL-827) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Ship v0.2 under the Metamorphic Labs proprietary license, with `basedpyright` out of the runtime dependencies, an sdist that holds only what is needed to build the package, and `orcapod.__version__`. - -**Architecture:** Packaging metadata only, plus a four-line `__version__` block in `src/orcapod/__init__.py`. Hatchling's sdist target is restricted with `only-include`; the wheel picks up `LICENSE` through hatchling's default `license-files`. Tests build the real distributions with `uv build` and pin their top-level contents. - -**Tech Stack:** hatchling + hatch-vcs, uv 0.13, pip-licenses, pytest. - ---- - -## Evidence gathered before planning - -- `pyproject.toml` declares `license = { text = "MIT License" }` and the classifier `License :: OSI Approved :: MIT License`. No LICENSE file exists. README.md, docs/, mkdocs.yml and CHANGELOG.md contain no MIT or open-source claims (`grep -rniE '\bMIT\b|open[- ]source|licen[cs]e'`); the only MIT statements are the two lines in `pyproject.toml` (plus historical `superpowers/` plans, which are records and stay untouched). -- No vendored third-party code with its own license notice: `grep -rliE 'license|copyright' src/` is empty. -- Baseline build (`uv build`, version `0.1.3.dev30+g07e5a2ea3`): sdist 3,365,148 bytes containing every repo top-level entry (`tests/`, `superpowers/`, `docs/`, `notebooks/`, `CLAUDE.md`, `.github/`, `examples/`, `bench/`, `design/`, `uv.lock`, ...); wheel 592,529 bytes with top level `orcapod/` and `orcapod-.dist-info/`, no LICENSE. -- License checks: `pip-licenses` audits the installed `orcapod` itself (it reports `orcapod 0.1.3.dev30+g07e5a2ea3 MIT License`). The repo's `_license-check.yml` and the org's required `metamorphic-labs/shared-workflows` `ci-license-check.yml` both run `--allow-only ... --partial-match`. The org allow-list ends with `LicenseRef-Metamorphic-Proprietary`; a plain `Proprietary` would fail it. -- Coordinator decision (ask, 2026-11-09): declare `LicenseRef-Metamorphic-Proprietary` in the prevailing org form, keep the LICENSE text exactly as the template's, and add the identifier to the repo's own `--allow-only` list (one line). Prevailing form across org repos (`gh search code`): `license = {text = "LicenseRef-Metamorphic-Proprietary"}` in 6 of 9 repos (3 use the PEP 639 string form). -- `pip-licenses` defaults to `--from=mixed`, which prefers trove classifiers over the `License` metadata field. A `License :: Other/Proprietary License` classifier would therefore make pip-licenses report `Other/Proprietary License` and fail both allow-lists. Task 1 verifies this empirically; if confirmed, the classifier is not added (it "does not apply" because it would break the required check) and that is reported. -- Runtime dependencies with no import in `src/`: `basedpyright` (dev tool, moved), `pymongo` (nothing depends on it), `pyyaml`, `typing_extensions` (transitive anyway). `s3fs` (fsspec plugin UPath needs for `s3://`), `tzdata` (pandas needs it), `gitpython` and `typer` (imported) are genuine runtime deps. ~~Coordinator decision: keep `pymongo`, `pyyaml`, `typing_extensions`; report as follow-up.~~ Superseded by the developer decision in the addendum below: `pymongo` and `pyyaml` are removed, `typing_extensions` moves to the dev group. -- hatch-vcs generates `src/orcapod/_version.py` (gitignored) defining `__version__: str`. -- CI runs `uv run pytest -v -m "not postgres"`, with `uv` on PATH and `fetch-depth: 0`. - -## Files - -- Create: `LICENSE` -- Modify: `pyproject.toml` (`license`, classifiers, dependency lists, `[tool.hatch.build.targets.sdist]`) -- Modify: `uv.lock` (basedpyright move; per the addendum, also the `pymongo`/`pyyaml` removal and the `typing_extensions` move) -- Modify: `.github/workflows/_license-check.yml` (append the identifier to `--allow-only`) -- Modify: `src/orcapod/__init__.py` (`__version__` block and `__all__` entry) -- Modify: `tests/test_packaging.py` (new tests) -- Modify: `DESIGN_ISSUES.md` (one entry appended at the end) - -### Task 1: License - -- [ ] **Step 1: Failing tests** appended to `tests/test_packaging.py`: - -```python -LICENSE_ID = "LicenseRef-Metamorphic-Proprietary" - - -def test_license_is_metamorphic_proprietary(): - """Orcapod is proprietary to Metamorphic Labs from v0.2 on (ITL-827).""" - project = tomllib.loads(PYPROJECT.read_text())["project"] - assert project["license"] == {"text": LICENSE_ID} - assert not [c for c in project.get("classifiers", []) if "OSI Approved" in c] - assert (REPO_ROOT / "LICENSE").read_text().startswith( - "Copyright (c) Metamorphic Labs, Inc. All Rights Reserved." - ) -``` - -- [ ] **Step 2:** `uv run pytest tests/test_packaging.py -v` → FAIL (`MIT License` != ...). -- [ ] **Step 3:** Create `LICENSE` byte-identical to the template (`gh api repos/metamorphic-labs/python-project/contents/template-python/LICENSE -q .content | base64 -d > LICENSE`). In `pyproject.toml` set `license = { text = "LicenseRef-Metamorphic-Proprietary" }` and remove the MIT classifier. Empirically check whether adding `License :: Other/Proprietary License` changes what pip-licenses reports (re-sync, run `uv run pip-licenses --packages orcapod`); keep it only if both license checks still pass. -- [ ] **Step 4:** Append `;LicenseRef-Metamorphic-Proprietary` to the `--allow-only` value in `.github/workflows/_license-check.yml`. -- [ ] **Step 5:** Tests pass. Commit `feat(packaging): license Orcapod as Metamorphic Labs proprietary (ITL-827)`. - -### Task 2: basedpyright to the dev group - -- [ ] **Step 1: Failing test:** - -```python -def test_basedpyright_is_a_dev_dependency_only(): - """A type checker is a development tool, not something users install (ITL-827).""" - data = tomllib.loads(PYPROJECT.read_text()) - runtime = [d.lower() for d in data["project"]["dependencies"]] - dev = [d.lower() for d in data["dependency-groups"]["dev"]] - assert not [d for d in runtime if d.startswith("basedpyright")] - assert [d for d in dev if d.startswith("basedpyright")] -``` - -- [ ] **Step 2:** FAIL. **Step 3:** move `"basedpyright>=1.38.1"` into `[dependency-groups] dev` (alphabetical slot after `httpie`? the list is roughly alphabetical; place it first, before `httpie`). Run `uv lock` (no `--upgrade`), then compare `git diff uv.lock`: keep only the hunks for the `orcapod` package's `dependencies` / `requires-dist` / `dev` group; hand-revert any lockfile `revision` bump or marker reordering; confirm `uv lock --check` exits 0. -- [ ] **Step 4:** Tests pass. Commit `fix(deps): move basedpyright from runtime to dev dependencies (ITL-827)`. - -### Task 3: sdist contents - -- [ ] **Step 1: Failing tests** (module-scoped fixture builds once; skip only if `uv` is absent): - -```python -@pytest.fixture(scope="module") -def built_dists(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: - uv = shutil.which("uv") - if uv is None: - pytest.skip("uv is required to build the distributions") - out_dir = tmp_path_factory.mktemp("dist") - subprocess.run([uv, "build", "--out-dir", str(out_dir), str(REPO_ROOT)], check=True, capture_output=True) - return next(out_dir.glob("*.tar.gz")), next(out_dir.glob("*.whl")) - - -def test_sdist_contains_only_build_inputs(built_dists): top-level names == {"LICENSE", "PKG-INFO", "README.md", "pyproject.toml", "src"}; src/ holds only orcapod/ -def test_wheel_top_level(built_dists): top-level == {"orcapod", "orcapod-.dist-info"} -def test_license_shipped_in_sdist_and_wheel(built_dists): LICENSE bytes in both == repo LICENSE -``` - -(`uv build` builds the sdist and then the wheel *from the sdist*, so the wheel test also proves the trimmed sdist is buildable.) - -- [ ] **Step 2:** FAIL (sdist holds tests/, superpowers/, ...; LICENSE absent before Task 1 is in the build). -- [ ] **Step 3:** add to `pyproject.toml`: - -```toml -[tool.hatch.build.targets.sdist] -only-include = ["src/orcapod", "README.md", "LICENSE"] -``` - -- [ ] **Step 4:** Tests pass; record sizes. Commit `build(packaging): ship only package source, README and LICENSE in the sdist (ITL-827)`. - -### Task 4: `orcapod.__version__` - -- [ ] **Step 1: Failing tests** run in subprocesses (a meta-path finder blocks `orcapod._version`; no `sys.modules` manipulation): normal import returns `orcapod._version.__version__` and equals `importlib.metadata.version("orcapod")`; with `_version` blocked it falls back to the installed metadata version; with both unavailable it is `"0.0.0+unknown"`. -- [ ] **Step 2:** FAIL (`AttributeError: module 'orcapod' has no attribute '__version__'`). -- [ ] **Step 3:** top of `src/orcapod/__init__.py`: - -```python -try: - from ._version import __version__ -except ImportError: # source tree without the hatch-vcs generated _version.py - from importlib.metadata import PackageNotFoundError, version - - try: - __version__ = version("orcapod") - except PackageNotFoundError: - __version__ = "0.0.0+unknown" -``` - -and add `"__version__"` to `__all__`. Check ruff + basedpyright report nothing new on these lines. -- [ ] **Step 4:** Tests pass. Commit `feat: expose orcapod.__version__ (ITL-827)`. - -### Task 5: Design issue, verification, PR - -- [ ] Append one `DESIGN_ISSUES.md` entry (resolved) at the end. -- [ ] Full suite `uv run pytest tests/`. -- [ ] Build; clean venv `uv venv` + `uv pip install ` (no overrides); `uv pip check`; smoke pipeline (DictSource -> function pod) and print `orcapod.__version__`. -- [ ] License checks: repo command (`uv sync --locked --dev` + `_license-check.yml` command) and org command (`uv run --no-dev --with pip-licenses pip-licenses --allow-only= --partial-match --ignore-packages `), both exit 0; show the orcapod line. -- [ ] Rebase on origin/main, push, PR "Fixes ITL-827". - -## Addendum: developer decision on unused runtime dependencies (2026-11-09) - -The developer reversed the earlier "keep" decision: remove a runtime dependency that is unused -anywhere, and move one used only by tests to the dev group. - -Evidence (imports, `LazyModule`/`import_module` strings, entry points and plugin loading over -`src/`, `tests/`, `test-objective/`, `examples/`, `bench/`, `notebooks/`, `docs/`, `design/`; -reverse dependencies from `uv tree --all-groups --invert`): - -- `pymongo`: no reference anywhere; nothing else depends on it. Removed. -- `pyyaml`: imported only by two `notebooks/old_tutorials/` notebooks, which target a package - that no longer exists; `ray` (extra) and dev tools (pre-commit, mkdocs, omegaconf) declare it - themselves. Removed. -- `typing_extensions`: imported only by `tests/test_logical_types/test_protocols.py` - (`get_protocol_members`, stdlib only from 3.13); `pydantic` and others still pull it in at - runtime. Moved to the dev group. -- Kept, not clear-cut: `tzdata` (no import; time-zone database for `zoneinfo`/pyarrow on - systems without one, also required by pandas) and `s3fs` (no import; the fsspec - implementation `UPath` loads by name for `s3://` URIs). - -### Task 6: drop and move the unused dependencies - -- [ ] Failing tests `test_unused_package_is_not_a_dependency[pymongo|pyyaml]` and - `test_typing_extensions_is_a_dev_dependency_only` (PEP 503 name normalization). -- [ ] Edit `pyproject.toml`; `uv lock`; trim the lock to the `orcapod` block plus the removed - `pymongo` package block; `uv lock --check`. -- [ ] Re-run the clean install, smoke test and both license checks. diff --git a/superpowers/plans/2026-11-09-itl-832-mergejoin-arrow-types.md b/superpowers/plans/2026-11-09-itl-832-mergejoin-arrow-types.md deleted file mode 100644 index 9666b03e7..000000000 --- a/superpowers/plans/2026-11-09-itl-832-mergejoin-arrow-types.md +++ /dev/null @@ -1,160 +0,0 @@ -# ITL-832: MergeJoin keeps its inputs' Arrow types — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build every `MergeJoin` merged column (data and source-info) from the input columns' Arrow -element type, and keep the Arrow types of every pass-through column of `MergeJoin` and -`PolarsFilter`, so materialized output matches the inputs and the predicted schema with or without rows. - -**Architecture:** One construction for aggregated list columns lives in `arrow_utils` -(`aggregated_list_type` + `list_array`), used by `build_aggregated_table` (Batch, GroupBy) and -`MergeJoin`. Pass-through columns get their input Arrow types back after the Polars round-trip via a -new type-only `arrow_utils.restore_column_types` (nullability is ITL-820's and stays untouched). - -**Tech Stack:** pyarrow 23, polars, pytest, ruff, basedpyright; run everything with `uv run`. - -**Base:** PR #268's branch (stacked). Branch `eywalker/itl-832-mergejoin-merged-list-columns-must-keep`. - ---- - -## Decisions (coordinator, 2026-11-09, recorded from `ask`) - -- **Rule (developer):** respect the Arrow types the inputs have; infer from Python values only when - no Arrow type is known. -- **Q1 differing input layouts:** `MergeJoin.validate_binary_inputs` compares *Python* types, so - `int8`/`int64`, `string`/`large_string`, `float32`/`double`, `timestamp[ms]`/`[us]` collide today. - Use `pa.unify_schemas(..., promote_options="permissive")` for the element type when they differ; - identical types are used as-is; no new rejections. Document in the `MergeJoin` docstring; test each - promoted pair, including commutativity. The same promotion applies to shared tag keys. -- **Q2 outer layout:** `pa.list_(element)` (what `build_aggregated_table` already uses), through a - helper factored out of `build_aggregated_table`. Extension elements keep the `ListLogicalType` path. -- **Q3:** also fix `MergeJoin` non-colliding columns and `PolarsFilter` pass-through columns (Polars - turns `string`→`large_string`, `binary`→`large_binary`, `list`→`large_list`, struct child strings, - `fixed_size_binary[16]` record ids→`large_binary`). Layout only; content hashes unchanged (verify). -- **Q4 report only:** `Datagram.with_columns` (Index/Pick), barrier materializer, SemiJoin on list - columns. - -### Revision (developer, 2026-11-09, supersedes Q1 above) - -- **Q1 revised:** no promotion. Colliding data columns whose Arrow types differ (width, - `string`/`large_string`, timestamp unit, decimal precision) raise `InputValidationError` naming - the column and both types and telling the user to cast one side. Streams expose only Python - schemas, so the check runs in `binary_static_process` (execution), not in validation; build-time - validation still passes for such a pair. -- **Shared tag keys** whose Arrow types differ keep the joined type, as before (no promotion, no - rejection); tracked under ITL-835 / ITL-829. -- **Q3 condition:** content hashes must not change; proved by - `TestPassThroughContentHashes` and by the before/after snapshot against the base commit. -- **Q4:** Index/Pick/`Datagram.with_columns` is ITL-834 (another worker); findings handed over. - -## Verified facts - -- Arrow hasher is layout-insensitive (`string`==`large_string`, `list`==`large_list`) but - width-sensitive (`int8`≠`int64`, `float`≠`double`, decimal precision). -- `ArrowTableStream` normalizes the source-info table to large types, so merged `_source_*` columns - are `large_list` before and after. -- `pipeline_hash` derives from Python schemas: unaffected. -- Before snapshot saved by the scratch script `snapshot.py` (`before.json`) for the before/after report. - -## File map - -- `src/orcapod/utils/arrow_utils.py` — add `aggregated_list_type`, `list_array`, - `restore_column_types`; refactor `build_aggregated_table` to use `aggregated_list_type`. -- `src/orcapod/core/operators/merge_join.py` — delete `_inferred_type`, `_list_array`; snapshot input - types; build merged columns with the helpers; restore pass-through types at schema reconstruction. - Keep away from the `return ArrowTableStream(... tag_columns=...)` lines PR #272 edits. -- `src/orcapod/core/operators/filters.py` — `restore_column_types(filtered, table.schema)`. -- `tests/test_core/operators/test_merge_join_arrow_types.py` — new. -- `tests/test_core/operators/test_empty_inputs.py` — update PR #268's - `test_colliding_column_type_matches_non_empty_result` to assert the input-derived type. -- `tests/test_utils/` (or wherever arrow_utils tests live) — helper tests. -- `CLAUDE.md`, `.zed/rules` — "Arrow types" note; `DESIGN_ISSUES.md` — one entry at the end. - -### Task 1: arrow_utils helpers (TDD) - -- [ ] Write failing tests for `aggregated_list_type` (primitive → `pa.list_(t)`, extension → the - converter's `list[...]` extension type), `list_array` (empty and non-empty give the same type; - extension wraps storage), `restore_column_types` (types restored, nullability untouched, unknown - columns untouched). -- [ ] Run, see ImportError/AttributeError failures. -- [ ] Implement: - -```python -def aggregated_list_type(element_type, type_converter) -> pa.DataType: - if isinstance(element_type, pa.ExtensionType): - element_python_type = type_converter.arrow_type_to_python_type(element_type) - return type_converter.python_type_to_arrow_type(list[element_python_type]) - return pa.list_(element_type) - -def list_array(values, list_type) -> pa.Array: - if isinstance(list_type, pa.ExtensionType): - storage = pa.array(values, type=list_type.storage_type) - return pa.ExtensionArray.from_storage(list_type, storage) - return pa.array(values, type=list_type) - -def restore_column_types(table, reference_types: Mapping[str, pa.DataType]) -> pa.Table: - fields = [f.with_type(reference_types[f.name]) if f.name in reference_types else f - for f in table.schema] - return table.cast(pa.schema(fields, metadata=table.schema.metadata)) -``` - -- [ ] `build_aggregated_table` uses `aggregated_list_type`; Batch/GroupBy suites stay green. -- [ ] Commit `refactor(arrow_utils): share the aggregated list-type construction (ITL-832)`. - -### Task 2: MergeJoin merged columns (TDD) - -- [ ] New test module, parametrized over: int8, int16, int32, int64, uint8, float32, float64, string, - large_string, binary, large_binary, bool, timestamp[ms], date32, list, - large_list, decimal128(5,2), and a logical type (`Path` via a pod or the converter); - rows ∈ {0, 2}. Assert `x` type == `pa.list_(input)` (logical: converter's `list[...]`), and - `static_process(...).output_schema(columns=c) == op.output_schema(a, b, columns=c)` for - `c in ({}, {"system_tags": True})`. Promoted pairs (Q1) in both argument orders → same type. - Merged values still sorted, sources follow. -- [ ] Run: fails (e.g. `list` for int8). -- [ ] Implement in `binary_static_process`: snapshot `left_types`/`right_types` from `left_table`/ - `right_table` before the Polars join; `_merged_element_type(left, right)` = left if equal else - permissive unify; merged data and source arrays via `aggregated_list_type` + `list_array`. - Remove `_inferred_type`, `_list_array` and the inline ListLogicalType branch. -- [ ] Update PR #268's `test_colliding_column_type_matches_non_empty_result` to assert - `pa.list_(arrow_type)` too. -- [ ] Run merge-join + empty-input + logical-type tests; commit - `fix(merge_join): build merged columns from the inputs' Arrow element types (ITL-832)`. - -### Task 3: pass-through columns of MergeJoin and PolarsFilter (TDD) - -- [ ] Tests: non-colliding `string`, `binary`, `list`, `struct` data - columns, string tag, and system tag columns keep the input type through `MergeJoin`; same for - `PolarsFilter` (data, tag, system tags). Commutativity of shared tag key with - `string`/`large_string`. -- [ ] Run: fails (`large_string`). -- [ ] MergeJoin: build `passthrough_types` = every non-colliding, non-`_source_` column of - the two input tables (shared names → permissive unify); in the final schema block use these - types and `joined.cast(result_schema)`. -- [ ] PolarsFilter: `restore_column_types(filtered_table, {f.name: f.type for f in table.schema})`. -- [ ] Run operator suites; commit `fix(operators): keep pass-through Arrow types across Polars round-trips (ITL-832)`. - -### Task 4: docs and logs - -- [ ] "Arrow types" note in `CLAUDE.md` and `.zed/rules` (identical text). -- [ ] `DESIGN_ISSUES.md` entry at the end (resolved, Fix note). -- [ ] MergeJoin docstring: element type and promotion rule. -- [ ] Commit `docs: record the Arrow-type rule (ITL-832)`. - -### Task 5: verification and PR - -- [ ] Rerun `snapshot.py` → `after.json`; diff schemas / content hashes for the report. -- [ ] `uv run ruff check`, `uv run ruff format --check`, `uv run basedpyright` on changed files. -- [ ] Full suite on 3.11, 3.12, 3.13 (`uv run --python 3.x pytest tests/`). -- [ ] Rebase check vs origin/main (stacked on #268), push, PR (create-pr skill), link, worker_done. - -### Revision 2 (developer HOLD, 2026-11-09, supersedes the Q1 revision and the tag-key extension) - -- The rule for inputs whose Arrow types DISAGREE (colliding data columns and shared tag keys) is - paused. No rejection, no promotion: behaviour stays exactly as on the base commit (verified - identical on 64 cases), isolated in `merge_join._resolve_disagreeing_types` (TODO ITL-832), and - pinned by `TestDisagreeingArrowTypes` (which also passes against the base commit's code). -- The rejection implementation is kept on the local-only branch - `itl-832-local-reject-differing-arrow-types` (commit 71e707c2) so it can be restored. -- Everything undisputed stays: agreed types are respected for empty and non-empty results, the - shared list-construction helper, pass-through type restoration with the content-hash proof, and - the rule text. diff --git a/superpowers/plans/2026-11-09-itl-834-index-pick-arrow-types.md b/superpowers/plans/2026-11-09-itl-834-index-pick-arrow-types.md deleted file mode 100644 index 7a1e0fd5f..000000000 --- a/superpowers/plans/2026-11-09-itl-834-index-pick-arrow-types.md +++ /dev/null @@ -1,124 +0,0 @@ -# ITL-834: Index and Pick must preserve input Arrow types — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan -> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** After `Index` or `Pick`, every column the operator does not touch keeps the exact Arrow -type it had on input (tags and data, empty and non-empty results), and the extracted column's -Arrow type is derived from the input column's Arrow type instead of being inferred from Python -values. - -**Rule (developer, 2026-11-09):** whatever Arrow types the inputs have are what is respected. -Inferring an Arrow type from Python types is reserved for when no Arrow type is known. - -## Reproduction (main @ 07e5a2ea) - -Source: tag `tg: int16`; data `i8: int8, i16: int16, f32: float32, s: string, ls: large_string, -bn: binary, l8: list, p: orcapod.path, lst: list, d: dict[str,int]` (non-nullable). - -| Path | Result | -|---|---| -| `Index("lst", 0)` (in place) | untouched columns keep width; `lst` becomes `int64` (inferred from `int`), moved to last position | -| `Index("lst", 0, out="x")` | `i8/i16 -> int64`, `f32 -> double`, `l8/lst -> large_list`, `x: int64`; content hash of untouched data changes | -| `Index("lst", 5)` (all skipped) | tag `tg: int16 -> int64` and every data column widened | -| `Pick` | identical pattern for in place / `out=` / all skipped | - -Layout-only normalization (`string -> large_string`, `binary -> large_binary`, -`list -> large_list`) happens in `Datagram._init_from_table` for every operator and is out of -scope per the issue (no hash change). Tests compare against the large-normalized input type. - -## Root cause - -1. `Datagram.with_columns` and `Datagram.update` (`src/orcapod/core/datagrams/datagram.py`) - take the Arrow-native path only when the Arrow table is loaded **and** the dict cache is not. - `Index`/`Pick` read `data[column]` first (which loads the dict cache), so `with_columns` - falls into the dict path, which sets `_data_table = None` and later rebuilds the whole table - from the Python schema — every narrow width is lost. `update` has the same defect. -2. The Arrow path of `with_columns` passes a partial `column_types` as the full `python_schema`, - which silently drops new columns not named in `column_types` (latent; no src caller hits it). -3. `Index`/`Pick` pass the Python element type (`int`) for the extracted column, so it is inferred - as `int64` even when the input is `list`. -4. The all-skipped path in `Index`/`Pick` builds the empty table from the predicted *Python* - schema via `python_dicts_to_arrow_table`, so every column (tags included) is re-inferred. - -## Design - -- **Datagram (root fix):** `with_columns` and `update` operate on the Arrow table whenever it is - loaded (regardless of the dict cache); existing columns keep their Arrow fields verbatim; only - genuinely new columns get types, taken from `column_types` when given, otherwise inferred per - column from the value. The dict cache is invalidated after the change. Dict-only datagrams keep - their current Python-schema behaviour. -- **Extracted column type:** gate decision — see "Decision requested". Recommended: widen - `with_columns(column_types=...)` to accept `pa.DataType` entries as well as Python types; - Arrow entries are used verbatim. -- **Index/Pick:** derive the extracted Arrow type from the row's Arrow field - (`list/large_list/fixed_size_list -> value_type`; `map -> item_type`; - `list> -> value field type`; logical-type extension -> the element - logical type's Arrow type). In-place replacement keeps the column's position (`select` - original order after drop + add). The all-skipped path builds the empty table from the input - stream's Arrow schema (large-normalized) with only the target column's type replaced. - -## Decisions (coordinator, 2026-11-09) - -- Option A approved: `column_types` accepts `pa.DataType` entries on `Datagram`, `Data` and - `DatagramProtocol.with_columns` (protocol signature + docstring only). -- Gate #2 approved, each in its own commit with its own tests: - - **Meta:** `copy(include_cache=False)` (used by every mutator) dropped the meta Arrow - table, so narrow meta columns widened on any mutation. Carry it; meta mutators keep - untouched meta fields verbatim. - - **Stale Arrow schema:** dict-path `with_columns`/`select`/`drop`/`rename` copied the - cached `_data_arrow_schema`, so `arrow_schema()` disagreed with `as_table()` (and - `FunctionPodStream.as_table`, which builds tables from `arrow_schema()`, could drop a - column). Reset it. - -## Commits - -1. `fix(datagram)`: reset cached Arrow schema in dict-path mutators. -2. `fix(datagram)`: `with_columns`/`update` keep known Arrow types; `column_types` accepts - Arrow types. -3. `fix(datagram)`: keep meta Arrow types through every mutator. -4. `fix(operators)`: Index/Pick derive the extracted type, keep position, typed empty path. -5. `docs`: plan + DESIGN_ISSUES DG1. - -## Tasks - -- [ ] 1. Failing tests `tests/test_core/operators/test_index_pick_arrow_types.py`: Index and Pick, - in place / `out=` / all skipped / `fail_on_miss=True` (raises) ; untouched tag+data Arrow - types equal large-normalized input types; extracted type pinned (`int8` from `list`, - value type from dict); per-row content hash of untouched data equals input's; materialized - Python schema equals `output_schema()`; streaming (`async_execute`) path too. -- [ ] 2. Failing datagram tests `tests/test_core/datagrams/test_arrow_type_preservation.py`: - `with_columns`/`update` on an Arrow-backed datagram after the dict cache is loaded keep - `int8`/`float32`/`list`; partial `column_types` keeps all new columns. -- [ ] 3. Fix `Datagram.with_columns` / `update` (+ `Data.with_columns` signature). -- [ ] 4. Fix Index/Pick extracted type, in-place position, empty path. -- [ ] 5. Audit callers of datagram mutation methods across `src/`; record per caller. -- [ ] 6. Before/after probe (main vs branch): sources, FunctionPod, Join, Batch, GroupBy, - PipelineJob round trip through a DB — compare schemas and content hashes. -- [ ] 7. Full suite on 3.11/3.12/3.13; ruff + basedpyright on changed files; DESIGN_ISSUES entry. -- [ ] 8. Rebase, PR ("Fixes ITL-834", breaking-change note), worker_done. - -## Review round: operators use the stream's context (developer, 2026-11-10) - -Developer comments on PR 279 (`index.py:67`, `index.py:138`): `Index`/`Pick` cached -`self._converter = contexts.resolve_context(stream.data_context_key).type_converter`, -re-implementing context resolution that `stream.data_context` already does. Approved rule: -operators use the stream's context only, and nothing in `Index`/`Pick` reads a converter or -`self.data_context` after validation. - -`async_execute` has no stream, and `DataProtocol` exposes no converter. So the converter is used -once, in `validate_unary_input`, to resolve the fallback Arrow type of the extracted element (used -only for logical input columns such as `list[Path]` or `dict[str, Path]`). Only that resolved type -is cached, the same way `_output_type` is. If the eager conversion fails, validation does not -raise. The cached type stays `None`, and a `ValueError` naming the operator, column and output -type is raised only where the fallback is actually needed, which keeps the previous lazy -behaviour. - -- [x] Failing tests: Pick on `dict[str, Path]` (logical column), sync and async; Index on - `list[Path]`, async; deferred conversion error (validation passes, non-logical rows work, - logical rows raise a clear `ValueError`); no converter/context held after validation, and - the row-time code paths never touch a converter or `data_context`. -- [x] Refactor `Index` and `Pick`: drop `_converter`, its assert, `resolve_context` and the - `contexts` / `TypeConverterProtocol` imports if unused. -- [x] ruff, basedpyright (exit code), suite on 3.11/3.12/3.13; one commit; one push; reply on - both threads (do not resolve them). diff --git a/superpowers/plans/2026-11-09-plt-4816-docs-audit.md b/superpowers/plans/2026-11-09-plt-4816-docs-audit.md deleted file mode 100644 index 63edb3cf6..000000000 --- a/superpowers/plans/2026-11-09-plt-4816-docs-audit.md +++ /dev/null @@ -1,188 +0,0 @@ -# PLT-4816 (part 1): Make the Existing Documentation True — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Every Python sample in the user-facing docs runs against `main` and shows the output -the code really produces, the docs site builds under `mkdocs build --strict`, and a gap report -tells us what a workshop tutorial still has to cover. - -**Architecture:** A pytest module (`tests/test_docs/test_doc_samples.py`) extracts fenced -`python` blocks from the user-facing pages and executes each page as one session (blocks share a -namespace, top to bottom) in a **subprocess** with a throw-away `HOME` and working directory. -For every block it compares what the block prints against the `# ...` comments shown in the -block. A second test in the same directory runs `mkdocs build --strict`. Docs are then fixed -until both are green. No file under `src/` changes. - -**Tech stack:** pytest, stdlib only (`re`, `tokenize`, `subprocess`, `contextlib`), mkdocs-material -+ mkdocstrings (already in the `dev` dependency group). No new dependency. - -**Linear:** PLT-4816 (part of), ITL-807 (related; the strict extra-column behaviour stays for v0.2). - ---- - -## Scope - -In scope (owned): `docs/index.md`, `docs/getting-started.md`, `docs/concepts/*.md`, -`docs/api/*.md`, `docs/reference/*.md`, `README.md` prose and samples, `mkdocs.yml`, -`tests/test_docs/`, this plan, the gap report. - -Out of scope: `src/` (never edited), `docs/design`, `docs/plans`, `docs/specs`, -`docs/metamorphic`, `CHANGELOG.md`, `RELEASING.md`, version/install strings in `README.md` -(owned by ITL-810), the workshop tutorial itself. - -Decision gates (`ask` the coordinator first): restructuring the docs navigation, deleting a -page, adding a docs dependency. Library bugs found via a sample are **reported**, not papered -over. - -## File structure - -| File | Responsibility | -|---|---| -| `tests/test_docs/__init__.py` | package marker | -| `tests/test_docs/_doc_samples.py` | pure helpers: extract blocks, parse expected output, run a page (also the subprocess entry point) | -| `tests/test_docs/test_doc_samples.py` | unit tests for the helpers + one parametrized test per documentation page | -| `tests/test_docs/test_mkdocs_build.py` | `mkdocs build --strict` must exit 0 | -| `docs/**`, `README.md`, `mkdocs.yml` | fixes | -| `superpowers/plans/2026-11-09-docs-gap-report.md` | coverage gap report + proposed tutorial outline | - -## Harness contract (what a docs author needs to know) - -- Every ```` ```python ```` fence in a tested page is executed. Blocks on one page share one - namespace and run in order, so a later block may use names from an earlier one. -- A block is skipped only when the line directly above the fence (blank lines allowed) is the - HTML comment `` — invisible in the rendered site and on GitHub. Use it - only for samples that need infrastructure the test run lacks (a Postgres server), never to - hide a sample that is wrong or merely schematic: make such a sample runnable instead. -- Output check: every non-empty line the block prints must match some comment in that same - block (the text after `#`, whitespace-collapsed). `...` inside a comment is a wildcard, for - hashes and other run-dependent values. So a sample that shows `# {'bmi': 1.0}` while the code - prints `{'bmi': 2.0}` fails. Blocks that print nothing need no comments. -- The page runs with `HOME` and the working directory pointed at a temp directory, so samples - may write relative paths and `~/.orcapod` without touching the developer's machine. -- Tested pages: `README.md`, `docs/index.md`, `docs/getting-started.md`, `docs/concepts/*.md`, - `docs/reference/*.md`. - ---- - -### Task 1: Harness helpers (TDD) - -**Files:** create `tests/test_docs/__init__.py`, `tests/test_docs/_doc_samples.py`, -`tests/test_docs/test_doc_samples.py`. - -- [ ] **Step 1 — failing unit tests** for the helpers, in `test_doc_samples.py`: - - `extract_blocks(text)` returns `Block(lineno, source, skip)` for each `python` fence, ignores - `bash`/untagged fences, handles fences indented inside admonitions (dedents), and sets - `skip=True` only when preceded by ``. - - `expected_patterns(source)` returns comment texts via `tokenize` (a `#` inside a string - literal is not a comment). - - `unmatched_output(stdout, source)` returns `[]` when all printed lines are documented, - returns the offending line when a shown value is wrong, honours `...` wildcards, and - collapses runs of whitespace (pandas alignment). - - `run_blocks(blocks)` shares a namespace across blocks, reports an exception with the block's - line number, and keeps going after a failure. -- [ ] **Step 2 — run, expect ImportError/failures:** `uv run pytest tests/test_docs -q` -- [ ] **Step 3 — implement `_doc_samples.py`** (`extract_blocks`, `expected_patterns`, - `unmatched_output`, `run_blocks`, and `main(path)` that prints a JSON list of - `{lineno, status: passed|failed|skipped, error, unmatched}` — used as - `python -m tests.test_docs._doc_samples ` from the repo root). -- [ ] **Step 4 — run, expect unit tests green.** -- [ ] **Step 5 — commit:** `test(docs): add harness that executes documentation code samples` - -### Task 2: Per-page tests, baseline failure list - -- [ ] **Step 1** — add `DOC_PAGES` (the list under "Tested pages") and - `test_doc_page_samples_run(page, tmp_path)`: runs - `[sys.executable, "-m", "tests.test_docs._doc_samples", page]` with `cwd=tmp_path`, - `PYTHONPATH=:/src`, `HOME=tmp_path`, `ORCAPOD_HASH_CACHE_DB` unset; fails listing - every failed block as `page:line` with its error / undocumented output. Also - `test_doc_pages_exist` so a renamed page cannot silently drop out of the run. -- [ ] **Step 2** — run `uv run pytest tests/test_docs -q` and record the baseline: number of - blocks, number failing, why. This is the "show it fails before the fix" evidence and the input - for Task 3 and the gap report. - -### Task 3: Fix the pages - -For each failing block decide: **docs wrong** (fix sample/prose to match the code), **fragment** -(make it self-contained, or mark `docs-test: skip` if it needs external infrastructure), or -**library wrong** (leave the truthful behaviour out of the docs' happy path, record a minimal -reproduction for `worker_done`). Known items going in: - -- [ ] `docs/concepts/function-pods.md` — "Orcapod extracts the matching columns by name" is - false: extra data columns are rejected (ITL-807). Replace with the strict rule and a runnable - `SelectDataColumns` → pod example; fix the matching "determine which columns to read" bullet. -- [ ] `docs/concepts/function-pods.md` — `compute_bmi.pod(weight_stream, height_stream)` uses - undefined names: make it a runnable two-source example. -- [ ] `docs/concepts/function-pods.md` — `FunctionNode` sample: verify constructor arguments, - `run()`, `as_source()`, `get_all_records()` and show real output. -- [ ] `docs/concepts/streams.md` — "Core methods" fragments use an undefined `stream`; the - `all_info=True` column list is suspect (system-tag prefix is `_tag::`). -- [ ] `docs/concepts/operators.md` — column-selection and `PolarsFilter` samples use a `source` - that, at that point on the page, is the GroupBy source (no `weight`/`age`); `stream_a`, - `stream_b` undefined; verify Batch and GroupBy shown output. -- [ ] `docs/concepts/sources.md` — the `ColumnConfig` fragment uses `source` before it is - defined; check the "Concrete source types" table and `SourceStreamBuilder` claim against code. -- [ ] `docs/concepts/file-hash-caching.md` — `op.enable_file_hash_caching` is not exported from - `orcapod` (it lives in `orcapod.contexts`); absolute `/shared/...` paths; Postgres samples need - a server (skip marker). The page is also missing from the `mkdocs.yml` nav. -- [ ] `docs/concepts/identity.md` — the only sample is an untagged fence; make it a tested - `python` block that asserts the two hash claims. -- [ ] `docs/index.md`, `docs/getting-started.md` — verify every shown output; fix typos - ("transforms data data"); `walkerlab` → `nauticalab` repo URL in `docs/index.md` and - `mkdocs.yml` if the remote confirms it. -- [ ] `README.md` — has no Python samples today; prose only if wrong, and not the - release/version/install lines. -- [ ] Commit per page or small group: `docs(): ...`. - -### Task 4: API reference and strict build - -- [ ] **Step 1** — `uv run mkdocs build --strict`; list warnings. -- [ ] **Step 2** — fix: `:::` identifiers that no longer exist, broken relative links, pages - missing from nav (add `concepts/file-hash-caching.md` under Concepts — an addition, not a - restructure). If warnings come from the internal directories (`docs/design`, `docs/plans`, - `docs/specs`, `docs/metamorphic`), **ask** before excluding them from the build, since that - changes what the site publishes. -- [ ] **Step 3** — add names that exist in the public API but have no reference entry where it is - one line (`PipelineJob`, `side_effect_pod`/`sink_pod`, ...); larger additions go in the gap - report. -- [ ] **Step 4** — `tests/test_docs/test_mkdocs_build.py`: run - `[sys.executable, "-m", "mkdocs", "build", "--strict", "--site-dir", tmp_path]` from the repo - root, assert exit code 0, show the log tail on failure. Skip (with reason) only if `mkdocs` is - not importable. -- [ ] **Step 5** — commit: `docs(api): ...`, `test(docs): build the site in strict mode`. - -### Task 5: Small gaps - -- [ ] Add, only where it is a paragraph or one sample and the surrounding page already covers - the area: e.g. runnable `MergeJoin` / `SemiJoin` / `MapData` samples in `operators.md`, a - short "common errors" note where a page already describes the rule. Everything bigger is - listed in the gap report instead. - -### Task 6: Gap report - -- [ ] Write `superpowers/plans/2026-11-09-docs-gap-report.md`: for each step of the newcomer - path (install, first pipeline, sources, function pods, operators incl. Join/Batch/GroupBy, - running a `PipelineJob` with a database, caching and re-runs, inspecting results, file/path - types, common errors) — covered well / was wrong and is now fixed / missing — with evidence - (page and section), then the suspected library bugs with reproductions, then a proposed - outline for a self-paced workshop tutorial. Claims about library behaviour in the report must - come from code that was actually run. - -### Task 7: Finish - -- [ ] `DESIGN_ISSUES.md`: one self-contained entry appended at the end for the docs drift - (status `resolved`, **Fix:** note). -- [ ] `CLAUDE.md` + `.zed/rules`: one short "Documentation samples" instruction (how the - harness treats fences, the skip marker), updated in both files as CLAUDE.md requires. -- [ ] `uv run pytest tests/` — full suite; record passes, failures and skips by name. -- [ ] Rebase onto `origin/main` if it moved, rerun. -- [ ] Push, open PR against `main` (body: "Part of PLT-4816", "Related: ITL-807"), - `orca linear attach`, `orca worktree set --worktree current --pr `. -- [ ] `worker_done` with: samples run / fixed, harness path, mkdocs result, gap report path, - suspected library bugs, suite result, PR URL. - -## Self-review - -- Spec coverage: change items 1–4 map to Tasks 1–3, 4, 6, 5. Acceptance items map to Task 7. -- No library code changes; no dependency changes; nav only gains one existing page. -- Risk: subprocess-per-page costs a few seconds of import time per page (about ten pages). - Accepted for isolation — `enable_file_hash_caching` mutates process-global state. diff --git a/superpowers/plans/2026-11-09-pr-272-review-revision.md b/superpowers/plans/2026-11-09-pr-272-review-revision.md deleted file mode 100644 index d631e9d58..000000000 --- a/superpowers/plans/2026-11-09-pr-272-review-revision.md +++ /dev/null @@ -1,120 +0,0 @@ -# PR #272 review revision (ITL-817) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Address the developer's four inline comments on PR #272 with the revision plan the -developer approved: honest record-ID docs, one canonical column-order rule for commutative -joins, the order defined in one place, and an explicit pyright suppression. - -**Architecture:** Column order follows the operator's declared argument symmetry. Commutative -operators (`Join`, `MergeJoin`) list tag and data columns by first appearance across their -inputs in canonical `pipeline_hash` order; `output_schema` / `binary_output_schema` is the only -place that computes it and `static_process` reads it back. Ordered operators (`SemiJoin`) keep -operand order (it already returns the left stream's schema and table unchanged). - -**Tech Stack:** Python 3.11-3.13, PyArrow, Polars, pytest, uv, ruff, basedpyright. - -**Review comments:** 4520619036 (docs wording), 4520632441 (which order), 4520639575 (derive from -output schema), 4520654070 (suppress type warning). - ---- - -## Verified facts the plan relies on - -- `ArrowHasher.hash_table`, the semantic hash of a `Schema`, and the semantic hash of a `dict` - are all column/key-order-insensitive (checked with a scratch script). Record IDs - (`_build_record_id_preimage` hashes `tag.as_table(system_tags=True)` + input data hash), - `content_hash` and `pipeline_hash` therefore cannot change when column order changes. -- `ArrowTableStream.keys()` follows the `tag_columns` argument order, but - `ArrowTableStream.output_schema()` and `as_table()` follow the **physical** table order. So the - operator must physically reorder the table to the schema order, not just pass `tag_columns`. -- `Join.static_process` already joins in canonical order (`order_input_streams`); - `Join.output_schema` unions in call order (the mismatch the developer flagged). -- `MergeJoin.binary_static_process` joins in **call** order, and `binary_output_schema` builds - its data schema from a `set` (order depends on `PYTHONHASHSEED`). -- `Join` async: N>=3 already chains in canonical order; N=2 joins `inputs[0]` x `inputs[1]` in - call order. `MergeJoin` async is barrier mode via `static_process`. -- Downstream identities key on system tags + input data hash, so a moved row under a positional - record ID gets a new key: it is recomputed and appears as a new entry with the same user tag - values and a different record ID; nothing is paired with the wrong row's result. - -## Files - -- Modify: `src/orcapod/core/operators/join.py` (`output_schema`, `static_process`, async N=2) -- Modify: `src/orcapod/core/operators/merge_join.py` (`binary_output_schema`, `binary_static_process`) — minimal -- Modify: `docs/concepts/sources.md`, `docs/concepts/operators.md` -- Modify: `src/orcapod/core/sources/arrow_table_source.py`, `src/orcapod/core/sources/stream_builder.py` -- Modify: `superpowers/specs/2026-11-09-itl-817-content-derived-record-ids.md` (same lineage claim) -- Modify: `DESIGN_ISSUES.md` (RS1, RS2 of this PR's entry) -- Test: `tests/test_core/operators/test_join_tag_order.py` - -### Task 1: Failing tests for canonical column order - -- [ ] Rewrite `test_join_tag_order.py`: a subprocess script (run for `PYTHONHASHSEED` 1..6) that, - for `Join()` (left `{probe, x}`, right `{method, w}`) and `MergeJoin()` (left `{probe, x, y}`, - right `{method, x, z}`), records for both `op(a, b)` and `op(b, a)`: `stream.keys()`, - `op.output_schema(...)` keys with and without system tags, `as_table()` and - `as_table(all_info=True)` column names, first-row tag/data keys, `op.static_process(...) - .output_schema(...)` keys, and the rows of `as_table(all_info=True)` sorted. -- [ ] Assert: both operand orders give identical values for every field; prediction equals - `static_process` output schema order (with and without system tags); every seed agrees. -- [ ] In-process async test for `Join` (2 inputs, with `input_pipeline_hashes`): first-row tag - and data keys equal the sync stream's keys for both operand orders. -- [ ] Add the targeted `# pyright: ignore[]` on the `json.loads` return (rule name taken - from basedpyright's own report). -- [ ] Run `uv run pytest tests/test_core/operators/test_join_tag_order.py -v`; expect FAIL. - -### Task 2: Join - -- [ ] `output_schema`: iterate `self.order_input_streams(*streams)` instead of `streams`. -- [ ] `static_process`: delete the `tag_order` loop and the `tag_keys`-based reorder; replace with - -```python -tag_schema, data_schema = self.output_schema(*streams) -tag_order = list(tag_schema.keys()) -data_order = list(data_schema.keys()) -listed = {*tag_order, *data_order} -table = table.select( - tag_order + data_order + [c for c in table.column_names if c not in listed] -) -return ArrowTableStream(table, tag_columns=tuple(tag_order)) -``` - -- [ ] Async N=2: join `inputs` in canonical position order (shared helper with `_staggered_join` - that reads the canonical position from the suffix), passing the permuted suffixes. -- [ ] Comment states the rule: commutative operator, canonical `pipeline_hash` order, first - appearance; inputs with equal pipeline hashes keep call order (stable sort). - -### Task 3: MergeJoin (minimal) - -- [ ] `binary_output_schema`: iterate `self._canonical_order(...)`; tags by `union_schemas` of the - canonical first then second; data keys by first appearance (first's keys, then second's new - keys) instead of the `set`. -- [ ] `binary_static_process`: take `left_table`/`right_table` and their keys from the canonical - order (type converter stays the call-order left stream's, unchanged); replace the final - `all_tag_keys` reorder and `ordered_tag_keys` with the same schema-derived `select` as Join. - -### Task 4: Docs and docstrings - -- [ ] `docs/concepts/sources.md`, `ArrowTableSource`, `_make_provenance_token`, `DESIGN_ISSUES.md` - RS2, and the spec: positional default record ID; a moved row gets a different record ID, is - treated as a new entry (recomputed; tag table shows two entries with identical tag values and - distinct record IDs); no result is paired with the wrong row; `record_id_column` + explicit - stable `source_id` avoids the duplicates and the recomputation. Remove lineage wording and the - swap parenthetical. -- [ ] `docs/concepts/operators.md`: one sentence stating the column-order rule and that callers - should address columns by name; check the Join example output against a real run. -- [ ] `DESIGN_ISSUES.md` RS1 fix note: canonical order, single definition in `output_schema`. -- [ ] CLAUDE.md / `.zed/rules`: update only if they describe join column order (they do not at - plan time; re-check). - -### Task 5: Verify, commit, merge main, push, reply - -- [ ] Targeted tests pass; hash-stability suites pass unchanged - (`tests/test_hashing`, `tests/test_pipeline/test_system_tag_run_stability.py`, operator tests). -- [ ] One commit `fix(operators): ...` with Co-Authored-By trailer. -- [ ] `git merge origin/main` (merge commit), resolve `DESIGN_ISSUES.md` keeping both sides. -- [ ] ruff check, ruff format --check, basedpyright (exit code), pytest on 3.11/3.12/3.13, - `pytest test-objective/`. -- [ ] One push; reply on each of the four threads (no resolve); update PR description; watch CI - and CodeRabbit (max 3 rounds). diff --git a/superpowers/plans/2026-11-09-pr-275-coderabbit-review.md b/superpowers/plans/2026-11-09-pr-275-coderabbit-review.md deleted file mode 100644 index 4e703d8e0..000000000 --- a/superpowers/plans/2026-11-09-pr-275-coderabbit-review.md +++ /dev/null @@ -1,59 +0,0 @@ -# PR #275 — CodeRabbit review log - -PR: https://github.com/metamorphic-labs/orcapod-python/pull/275 (ITL-825) - -Every CodeRabbit finding is verified against the code before acting. Verdicts: -FIX (correct), PARTIAL (right problem, different remedy), DISMISS (wrong, out of -scope, or contradicts a project rule). - -## Findings - -| Round | Finding | Location | Verdict | Reason | Commit / reply | -|---|---|---|---|---|---| -| 1 (review of d4b30454) | `stabilize_type_names` replaces names one at a time, so a shorter name can rewrite the start of a longer one; sort replacements by length | `src/orcapod/utils/canonical_type_names.py:140-146` | PARTIAL | The cited case (outer class + nested class) already gives the right output: both resolve to the same public module. The real defect is a resolved name that is a prefix of a *different, unresolved* name (`json._impl._Exported` inside `json._impl._ExportedExtra`), or a name inside a longer dotted path. Sorting by length does not fix that, because the unresolved name never enters the replacement map. Fixed with one regex pass over whole dotted names, longest first. Reproduced by 2 failing tests first. Output for every real type is unchanged (438-value probe identical on 3.11/3.12/3.13). | bb25648e, thread 4520061937 | -| 1 | Ruff F401: unused `types` import | `tests/test_core/sources/test_sources_comprehensive.py:275` | DISMISS | Pre-existing on `main` (07e5a2ea) and not on a line this PR touches. Repo-wide ruff debt is being fixed separately on `main` (ITL-826); fixing it here would conflict with that work. | thread 4520061943 | -| 1 | Ruff F841: unused local `combined` | `tests/test_hashing/test_hash_utils.py:268` | DISMISS | Same: pre-existing on `main` (line 264 there), not touched by this PR, and owned by ITL-826. | thread 4520061948 | -| 1 | Pre-merge checks (Description, Title, Linked Issues, Out of Scope, Docstring Coverage 81.25%) | summary comment | — | All passed; nothing to do. | round summary comment | - -## Coordinator items (not CodeRabbit) - -| Item | Verdict | Notes | Commit | -|---|---|---|---| -| README line 14 still says Python 3.11 and 3.12 | Deferred | PR #269 owns README version text; not edited here. | — | -| 72 new basedpyright warnings in files this PR touches (org typecheck fails on warnings) | FIX | Typing-only changes; canonical strings and hashes byte-identical (438-value probe on 3.11/3.12/3.13). See counts below. | d4b30454 | - -### basedpyright per-file counts (errors + warnings) - -New files must be 0; modified files must not exceed `main`. - -| File | main | before | after | -|---|---|---|---| -| src/orcapod/utils/canonical_type_names.py (new) | — | 10 | 0 | -| tests/test_hashing/test_canonical_type_names.py (new) | — | 41 | 0 | -| src/orcapod/core/data_function.py | 183 | 183 | 181 | -| src/orcapod/core/data_function_proxy.py | 79 | 79 | 79 | -| src/orcapod/core/sources/list_source.py | 49 | 53 | 49 | -| src/orcapod/hashing/hash_utils.py | 49 | 53 | 20 | -| src/orcapod/hashing/semantic_hashing/builtin_handlers.py | 131 | 131 | 131 | -| src/orcapod/hashing/semantic_hashing/semantic_hasher.py | 78 | 78 | 76 | -| src/orcapod/utils/function_info.py | 48 | 50 | 33 | -| tests/test_core/data_function/test_data_function_proxy.py | 21 | 26 | 21 | -| tests/test_core/sources/test_sources_comprehensive.py | 163 | 165 | 163 | -| tests/test_hashing/test_hash_utils.py | 187 | 187 | 116 | - -Remedies used: - -- `typing.cast(object, ...)` around `Any`-typed stdlib members (`Parameter.annotation`, - `Signature.return_annotation`, `__annotations__` items) before passing them to - `stabilize_type_names`. A declared `object` type does not silence `reportAny` on a member - access, while `cast` does. -- `Callable` → `Callable[..., object]` and `-> list` → `-> list[str]` on the hashing helpers - this PR already touches. -- Test parameters annotated; the implicit string concatenation is parenthesised. -- Narrow ignores, each with a stated reason: `reportDeprecated` on the four `typing.Optional`/ - `List`/`Sequence`/`Union` entries (the legacy forms are the inputs under test); - `reportPrivateUsage` on `ListSource._hash_tag_function` (the pre-hash string is the subject); - `reportUnusedParameter` on the `stem` fixture function. In - `test_canonical_type_names.py` the rule `reportUnusedParameter` is disabled file-wide, - because the golden functions' source text is hashed: a line-level comment inside them - changed `function_content_hash` (verified). diff --git a/superpowers/plans/2026-11-09-v0.2-release-checklist.md b/superpowers/plans/2026-11-09-v0.2-release-checklist.md deleted file mode 100644 index 15aa233b0..000000000 --- a/superpowers/plans/2026-11-09-v0.2-release-checklist.md +++ /dev/null @@ -1,554 +0,0 @@ -# Orcapod Python v0.2.0 — release checklist (ITL-810) - -Prepared 2026-11-09 against `main` at `61d0f18e`; updated 2026-11-10 after every selected -v0.2 PR merged (`main` at `c5ab1e05`). This is the ordered list of steps to cut `orcapod` -0.2.0, followed by the ITL-810 cut checklist with the evidence for each item. - -Nothing here has been triggered. The Release workflow has not been run and no tag has been -created. Every PR selected for 0.2.0 is merged except this one (#269), including the MIT -license (#283, ITL-875) and the golden hash regression suite (#284, ITL-626). **This PR -merged is the last precondition of the cut.** - -**Developer decisions since this checklist was first written (2026-11-09):** - -- **Distribution reversed (2026-11-10).** Orcapod is MIT licensed starting with 0.2.0, is - published on public PyPI as `orcapod`, and the repository `metamorphic-labs/orcapod-python` - will become public (ITL-875). This replaces the 2026-11-09 decisions (private - repository, proprietary license, internal registry only), which are void. The license - files change in #283; `.github/workflows/release.yml` is not changed. -- **0.2.0 is tag-only (developer, 2026-11-10).** Nothing is published for 0.2.0: the - coordinator creates the `v0.2.0` tag on the release commit after this PR merges, and - 0.3.0 is the first release published to PyPI from this repository. The public release - route (trusted publishers, rehearsal; ITL-875) is prepared for 0.3.0. -- Python 3.13 is supported (ITL-825, #275). It landed on `main` through #276. -- The organization's required CI (`ruff`, `ruff format`, `basedpyright`, `pytest` on Python - 3.11, 3.12 and 3.13) is met by ITL-826 (#276, merged 2026-11-10 as `7dec4b54`, which - also contains #275). -- Operator outputs keep their inputs' Arrow types, normalized to large variants: - `MergeJoin` (with zero-row inputs), `PolarsFilter`, `Batch` and `GroupBy` (ITL-813 and - ITL-832, #278, merged 2026-11-10 as `ab544982`; #268 was closed as superseded by it), - and `Index` / `Pick` (ITL-834, #279, merged 2026-11-10 as `fde06d20`). Both change some - row content hashes and have entries under *Breaking changes and migration*. -- ITL-837, ITL-845 and ITL-849 were added to 0.2.0 on 2026-11-10 and merged the same day - (#280, #282, #281). Type strictness applies everywhere: no implicit widening; disagreeing - Arrow types are an error naming the column and both types. -- ITL-604 (#248, `NodeConfig.missing_cache_policy`) moves to v0.3. -- The golden hash regression suite (ITL-626, #284) is part of 0.2.0 (developer, - 2026-11-10). Hash values must stay identical across releases. - ---- - -## Part 1 — Before the cut - -### 1. Merge the open PRs — not done: this PR remains - -Rows 1–14 are settled by 2026-11-10 (merged, or for #268 closed as superseded by #278); the -table keeps the planned order and gives each merge commit. This PR (#269, row 15) remains; -this step is done when it has merged. - -| Order | PR | Issue | Why this position | -|---|---|---|---| -| 1 | [#267](https://github.com/metamorphic-labs/orcapod-python/pull/267) | ITL-134 — remove `pygraphviz`, restore green CI | **Merged 2026-11-09 (`07e5a2ea`).** Must be first. `pygraphviz` no longer builds on the `ubuntu-latest` image, so CI is red on every PR and the Release workflow's own test job would fail. Nothing else can show green CI until this is in. | -| 2 | [#276](https://github.com/metamorphic-labs/orcapod-python/pull/276) | ITL-826 — meet the organization's required CI (contains #275, ITL-825 — Python 3.13) | **Merged 2026-11-10 (`7dec4b54`).** Every later PR is checked by the required CI. | -| 3 | [#268](https://github.com/metamorphic-labs/orcapod-python/pull/268) | ITL-813 — `MergeJoin` on empty input | **Closed 2026-11-10, superseded by #278**, which contains its commits. | -| 4 | [#278](https://github.com/metamorphic-labs/orcapod-python/pull/278) | ITL-813 + ITL-832 — `MergeJoin` zero-row inputs; operator outputs keep the inputs' Arrow types, large-normalized | **Merged 2026-11-10 (`ab544982`).** Changes some row content hashes. | -| 5 | [#270](https://github.com/metamorphic-labs/orcapod-python/pull/270) | ITL-816 — `iter_data` drops rows on mismatched chunk layouts | **Merged 2026-11-10 (`cb3eed9a`).** Release fix. | -| 6 | [#279](https://github.com/metamorphic-labs/orcapod-python/pull/279) | ITL-834 — `Index` / `Pick` keep the input Arrow types | **Merged 2026-11-10 (`fde06d20`).** Changes row content hashes for narrow-typed columns. | -| 7 | [#272](https://github.com/metamorphic-labs/orcapod-python/pull/272) | ITL-817 — deterministic `Join` / `MergeJoin` column order; buffer materialization aligns by name | **Merged 2026-11-10 (`365a5f74`).** No hash or stored-value change; see step 3. | -| 8 | [#277](https://github.com/metamorphic-labs/orcapod-python/pull/277) | ITL-827 — proprietary license, runtime dependency cleanup, trimmed sdist, `orcapod.__version__` | **Merged 2026-11-10 (`8b92e608`).** Its license is replaced by MIT in #283 (row 13); the packaging cleanup stays. | -| 9 | [#271](https://github.com/metamorphic-labs/orcapod-python/pull/271) | PLT-4816 — docs audit | **Merged 2026-11-10 (`3a68ce93`).** Docs only. Its `docs/index.md` installation wording (internal package index) is replaced by the PyPI install commands. | -| 10 | [#281](https://github.com/metamorphic-labs/orcapod-python/pull/281) | ITL-849 — `GroupBy` / `Batch` after `MergeJoin`, `Batch` or `GroupBy` raised `ArrowTypeError` | **Merged 2026-11-10 (`62a182d7`).** No hash change for pipelines that already worked. | -| 11 | [#282](https://github.com/metamorphic-labs/orcapod-python/pull/282) | ITL-845 — materializing rows into a stream stamped row 0's source info on every row (`DESIGN_ISSUES.md` O3) | **Merged 2026-11-10 (`f18339e3`).** Behaviour fix only: no version bump, stored rows not repaired (ITL-853). | -| 12 | [#280](https://github.com/metamorphic-labs/orcapod-python/pull/280) | ITL-837 — `Join`'s async path keeps Arrow types; type-strict shared keys; with ITL-854 / ITL-857 (`Join` drops meta columns) and ITL-859 (`Datagram.copy` schema) | **Merged 2026-11-10 (`c438d4d1`).** Async `Join` row hashes change for narrow columns (now equal to sync). | -| 13 | [#283](https://github.com/metamorphic-labs/orcapod-python/pull/283) | ITL-875 — MIT license from 0.2.0 | **Merged 2026-11-10 (`4edfde8e`).** `LICENSE` (copyright NauticaLab), `license = "MIT"`, `tests/test_packaging.py`. | -| 14 | [#284](https://github.com/metamorphic-labs/orcapod-python/pull/284) | ITL-626 — golden hash regression suite | **Merged 2026-11-10 (`c5ab1e05`).** Tests only; no hash changed. Added to 0.2.0 by the developer. | -| 15 | **this PR** (#269) | ITL-810 — release prep | Open. Changelog entries for rows 10–14 are in `[0.2.0]` (step 2). | - -Also merged: #266 (ITL-818, agent coordination runbook, `fd352956`), documentation for -contributors that does not affect the package. - -**Not part of 0.2.0** — leave open: #248 (ITL-604, `NodeConfig.missing_cache_policy`; moved to -v0.3), #258 (ITL-611, `set[T]` for primitives), #145 (PLT-1453, async connectors, changes -requested and conflicting). - -### 2. Finish `CHANGELOG.md` (in this PR) - -- [x] Replace `unreleased` in `## [0.2.0] — unreleased` with the release date. - *Done 2026-11-10: `## [0.2.0] — 2026-11-10`.* -- [x] Resolve the remaining placeholder lines (the ITL-134 ones when #267 merged; the - ITL-604 one removed because #248 moved to v0.3; ITL-825 / ITL-826 when #276 merged; - ITL-816 when #270 merged; the rest after everything else merged): replace each with - the final wording and PR number. - *Done 2026-11-10 after merging `main` at `ab544982`: every entry for ITL-813 and - ITL-832 (#278), ITL-834 (#279), ITL-817 (#272) and ITL-827 (#277) was written from - the merged PR and checked against `main`. A case-sensitive search for the marker over - the tracked files finds none.* -- [x] Describe the public release route in `CHANGELOG.md`, `README.md`, `docs/index.md` - and `RELEASING.md`: MIT, `pip install orcapod` / `uv add orcapod`, and release by - manual dispatch of `release.yml` (TestPyPI, then PyPI, through OIDC trusted - publishing). *Done 2026-11-10; every internal-registry, not-on-PyPI and proprietary - sentence is gone.* -- [x] **Tag-only path chosen (developer, 2026-11-10).** *Done in this PR: the - `CHANGELOG.md` *Distribution* paragraph, `README.md` and `docs/index.md` say `0.2.0` - is a tag only and `pip install orcapod` installs `0.1.4`; the notices merge with this - PR, before the tag.* The item as written: - If 0.2.0 is **not** published to PyPI (step 6), say so everywhere the install - command appears, because `pip install orcapod` would then install `0.1.4`: - - `CHANGELOG.md`: follow the `RELEASE-DAY` comment, so the *Distribution* paragraph - says `0.2.0` is a tag only. - - `README.md` and `docs/index.md`, *Installation*: add after the install commands - "`0.2.0` is not on PyPI, where the latest release is `0.1.4`; `0.2.0` is the tag - `v0.2.0` of the repository." Do not give a git install URL while the repository is - not public. - - Merge these notices to `main` **before** the tag is created. If they merge after the - step 4 checks, repeat step 4 on the new `main` commit, and tag that checked commit. -- [x] **Move entries that other PRs added.** #267 adds its entry under what was the - `[Unreleased]` heading. This PR renames that heading to `[0.1.1]`, so after the - rebase #267's entry will sit in the wrong release. Move it (and any entry the - three fix PRs added the same way) into `[0.2.0]`. Expect a merge conflict in - `CHANGELOG.md`, `README.md` and possibly `docs/index.md` / `pyproject.toml`, which - #267 also edits. - *Done for #267 on 2026-11-09 in merge commit `5982f099`: the merge was textually - clean, and its entry now sits under `[0.2.0]` > *Changed*. Done for #278 on - 2026-11-10: git placed its entry under `[0.1.1]`; it is folded into `[0.2.0]` > - *Breaking changes and migration*, so each change is described once. Done for #280, - #281 and #282 on 2026-11-10 when merging `main` at `c438d4d1`: the ITL-837, - ITL-854/ITL-857 and ITL-845 entries are under *Breaking changes and migration* - (ITL-845 with the developer's wording on stored rows unchanged), ITL-849 and ITL-859 - under *Fixed*, each once.* -- [x] Delete the `` comment. *Done 2026-11-10.* - -### 3. Re-check what the in-flight fixes do to stored data - -- [x] ITL-817 (#272) changes no system tag value, `record_id`, `content_hash` or - `pipeline_hash`; only column order. It has an entry under *Breaking changes and - migration* for code that reads columns by position, a *Fixed* line, and a *Known - limitations* item for positional default record IDs. -- [x] ITL-834 (#279) and ITL-813 + ITL-832 (#278): the stored Arrow types and row content - hashes they change are stated under *Breaking changes and migration*. Checked on - 2026-11-10 by running the same pipelines on the pre-merge commits (`fde06d20`, - `365a5f74`) and on `ab544982`: row data hashes change only where stated; tag hashes, - stream `content_hash` and `pipeline_hash` are unchanged. -- [x] ITL-837 (#280): async `Join` row hashes over narrow columns change to equal the sync - values; sync hashes, stream hashes and pipeline hashes are unchanged. ITL-854 / ITL-857 - change no hash, record ID or stored table. ITL-849 (#281) changes no hash for - pipelines that already worked. ITL-845 (#282) changes only `_source_*` values (and an - operator node's per-row `_record_hash`); stored rows are not repaired, and the - changelog advises dropping pipeline tables and keeping result tables (ITL-853). All - stated under *Breaking changes and migration* or *Fixed*. - -### 4. Re-run the release checks on the final `main` - -- [x] **Precondition: ITL-837 is merged** to `main` (#280, `c438d4d1`), with its - `CHANGELOG.md` entry under `[0.2.0]`; *Known limitations* no longer lists it. -- [x] **Precondition: ITL-845 is merged** to `main` (#282, `f18339e3`), with its - `CHANGELOG.md` entry under `[0.2.0]`. -- [x] **Precondition: ITL-849 is merged** to `main` (#281, `62a182d7`), with its - `CHANGELOG.md` entry under `[0.2.0]`. -- [x] **Precondition: the MIT license PR is merged** to `main` (#283, `4edfde8e`). -- [x] **The golden hash suite is on `main`** (#284, `c5ab1e05`), with its `CHANGELOG.md` - entry under `[0.2.0]` > *Changed*. - -The checks in Part 3 were run on `61d0f18e` with `pygraphviz` excluded, because this -machine has no Graphviz headers. Once everything above is merged: - -- [ ] **Golden hash tests pass** on the `main` commit you will release, on Python 3.11, 3.12 - and 3.13: `uv run --python pytest tests/test_hashing/golden -q` (644 tests, 608 - pinned values, on 2026-11-10). A failure means a hash moved since the values were - pinned: stop the release and treat it as a bug in the change that moved it. Never - regenerate the golden files to make this pass (`CLAUDE.md`, *Golden hash values*). -- [ ] CI on the `main` commit you will release: `test (3.11)`, `test (3.12)`, - `test (3.13)`, `license-check`, `objective-test` green, and the organization's - required checks (since #276): `ruff`, `ruff format`, `basedpyright`, `pytest` on - 3.11, 3.12 and 3.13. -- [ ] **Unmodified clean install.** This is the check that could not be done before - ITL-134 merged, and it must pass before publishing: - - The block runs in a subshell with `set -e`, so the first failing command stops it, and - every run uses fresh temporary directories, so a leftover environment can never stand in - for a clean install. The wheel is built from a clean, detached worktree at - `origin/main`, so local commits or uncommitted changes cannot leak into it; the block - prints the commit it built, which must be the commit you release. The pipeline result is - asserted, so a wrong result fails the block instead of reaching `SMOKE OK`. - - ```bash - ( - set -euo pipefail - git fetch origin main - work=$(mktemp -d) - git worktree add --detach "$work/src" origin/main - trap 'git worktree remove --force "$work/src"' EXIT - echo "Building $(git -C "$work/src" rev-parse HEAD)" - uv build --out-dir "$work/dist" "$work/src" - for v in 3.11 3.12 3.13; do - env_dir="$work/smoke-$v" - uv venv --python "$v" "$env_dir" - py="$env_dir/bin/python" - uv pip install --python "$py" "$work"/dist/orcapod-*.whl - uv pip check --python "$py" - (cd "$work" && "$py" -c " - from orcapod import function_pod - from orcapod.sources import DictSource - src = DictSource(data=[{'name': 'Alice', 'age': 30}], tag_columns=['name'], source_id='people') - @function_pod(output_keys='birth_year') - def f(age: int) -> int: - return 2026 - age - rows = [(t.as_dict(), d.as_dict()) for t, d in f.pod(src).iter_data()] - print(rows) - assert rows == [({'name': 'Alice'}, {'birth_year': 1996})], rows - ") - "$env_dir/bin/orcapod" --help - done - echo "SMOKE OK in $work" - ) - ``` - - Expected: no `--override`, no `--no-install-package`; the printed commit is the one - you release; `uv pip check` reports no incompatibility; the pipeline prints - `[({'name': 'Alice'}, {'birth_year': 1996})]`; the block ends with `SMOKE OK`. - (Before ITL-134, `uv pip check` reported `orcapod requires pygraphviz>=1.14, but it's - not installed`.) - - **Result, 2026-11-09 — pass, on this branch after merging `main` (`07e5a2ea`), merge - commit `5982f099`.** macOS arm64, uv 0.13.7. Wheel `orcapod-0.1.3.dev33+g5982f0990` - installed into clean venvs with no `--override` and no `--no-install-package`, on - Python 3.11.18 and 3.12.15: `uv pip check` reported `All installed packages are - compatible` (65 packages, `pygraphviz` not among them); the pipeline printed - `[({'name': 'Alice'}, {'birth_year': 1996})]`; `orcapod --help` and - `orcapod migrate --help` exited 0. The box stays unticked: repeat on the final `main` - commit once the remaining fix PRs are in. - -### 5. Confirm the version is free - -- [ ] `git ls-remote --tags origin refs/tags/v0.2.0` prints nothing. -- [ ] `0.2.0` is on neither nor - . (Checked 2026-11-09: PyPI's latest - is `0.1.4`; TestPyPI's latest is `0.1.2`.) - -### 6. Decide whether 0.2.0 is published — decided: tag-only (the preconditions stay open for 0.3.0) - -The Release workflow is the only publishing route (`RELEASING.md`). The last successful run -was `v0.1.2` on 2026-07-11, from `nauticalab/orcapod-python`; one later run (2026-08-20) -failed in `Test (Python 3.11)`, fixed by #254, and #254 / #256 then re-pinned every Action. -`0.1.3` and `0.1.4` were not published through this workflow (PyPI has them; TestPyPI and -GitHub Releases do not). The workflow has never published from -`metamorphic-labs/orcapod-python`. 0.2.0 is published to PyPI only if **all** of these are -done in time: - -- [ ] **Open: the PyPI and TestPyPI trusted publishers point at this repository.** On - pypi.org and test.pypi.org, an owner of the `orcapod` project adds a GitHub trusted - publisher with owner `metamorphic-labs`, repository `orcapod-python`, workflow - `release.yml` and environment `pypi` / `testpypi`. Only the PyPI project owner can do - this. A publisher that names another owner is rejected at upload time, which fails - `publish-testpypi` after the tag has been pushed. (ITL-875) -- [ ] **Open: the `pypi` and `testpypi` environments exist** in - `metamorphic-labs/orcapod-python` → *Settings → Environments*, and the - `LINEAR_ACCESS_KEY` repository secret is set. If the secret is missing, `linear-sync` - and `linear-complete` fail but publishing is not blocked. -- [ ] **Open: the route is rehearsed** from this repository: a pre-release uploaded to - TestPyPI and installed in a clean environment (ITL-875). `release.yml` has no - TestPyPI-only mode: a dispatched pre-release such as `0.2.0rc1` also goes to PyPI after - TestPyPI succeeds, creates a GitHub Release and runs `linear-complete`, so the - rehearsal method is ITL-875's to choose. - -**Decided (developer, 2026-11-10): repository visibility is not a precondition for -publishing to PyPI.** Making the repository public is a separate developer action under -ITL-875, after its history scan. Two facts that hold either way: - -- The published files contain no internal process material. On 2026-11-10 (`41435ac1`) the - sdist held only `.gitignore`, `LICENSE`, `PKG-INFO`, `pyproject.toml`, `README.md` and - `src/orcapod/` (194 entries), and the wheel only `orcapod/` and its `dist-info`: no - `superpowers/`, `docs/`, `tests/`, `CLAUDE.md`, `DESIGN_ISSUES.md` or - `AGENT_COORDINATION.md`. Code comments in `src/orcapod/` cite internal issue IDs (ITL-, - PLT-, NPIPE-, ENG-), with no tracker links. To re-check on the release commit, run - `uv build`, then `tar tzf dist/*.tar.gz` and `unzip -l dist/*.whl`. -- The README and the package metadata link to the repository; those links do not resolve - for the public until the repository is public. - -**Decided (developer, 2026-11-10): tag-only.** 0.2.0 is not published; step 7 is not run, -and the coordinator tags the release commit after this PR merges. The boxes above stay open -for 0.3.0. - -**If any box above is still open at the cut, do not run step 7.** The coordinator creates the -`v0.2.0` tag on the release commit by hand, nothing is published, `CHANGELOG.md` says so -(step 2), and 0.3.0 is the first public release from this repository. A hand-made `v0.2.0` -tag makes a later Release run for `0.2.0` abort at *Guard against duplicate tag*; publishing -0.2.0 afterwards would need that tag deleted first, which is allowed only because nothing was -published under it. - ---- - -## Part 2 — The cut - -### 7. Trigger the workflow — only if every box in step 6 is ticked (not run for 0.2.0: tag-only) - -Do not create the tag by hand on this path: the `build` job creates and pushes it. - -**Keep `main` at the commit you checked in step 4 until `build` has pushed the tag.** The -workflow builds whatever `main` is when its jobs check it out, not the commit you checked. -Before triggering, confirm `git ls-remote origin refs/heads/main` prints that commit, and -merge nothing until `build` finishes. If `main` moves before the tag is pushed, cancel the -run and repeat step 4 on the new commit. If the tag was pushed on a different commit, cancel -the run and wait until it has finished. Then check -, -and the logs of `publish-testpypi` and `publish-pypi`. Only if neither index has `0.2.0` -and neither job uploaded anything, delete the tag (`git push origin :refs/tags/v0.2.0`) as -in "If a Run Fails Part-Way" in `RELEASING.md` and start again. If anything was published, -or you cannot tell whether an upload happened, the version is used: keep the tag. - -[Actions → Release → Run workflow](https://github.com/metamorphic-labs/orcapod-python/actions/workflows/release.yml) - -| Input | Value | -|---|---| -| Use workflow from | `main` | -| Version | `0.2.0` | -| Branch | `main` (the default) | - -What the run does, job by job (`.github/workflows/release.yml`): - -1. `validate-branch` checks that the branch exists on origin. -2. `test (3.11)`, `test (3.12)`, `test (3.13)` (`uv sync --locked --all-extras --dev`, then - `pytest -m "not postgres"`) and `license-check` run in parallel. -3. `build` strips a leading `v`, creates the local tag `v0.2.0`, runs `uv build`, aborts if - the tag already exists on origin, pushes the tag, and uploads `dist/` as an artifact. -4. `publish-testpypi` (environment `testpypi`) runs - `uv publish --publish-url https://test.pypi.org/legacy/ dist/*`, authenticating through - OIDC trusted publishing (`id-token: write`). -5. `publish-pypi` (environment `pypi`), after TestPyPI succeeds, runs `uv publish dist/*` - the same way, then creates the GitHub Release for `v0.2.0` with generated notes and the - dist files attached. -6. `linear-sync` runs after `build`, in parallel with publishing; `linear-complete` runs after - `publish-pypi` and `linear-sync` succeed. - -If the `testpypi` / `pypi` environments have required reviewers, approve the deployment -when the run pauses. - -**If a job fails**, follow "If a Run Fails Part-Way" in `RELEASING.md`. In short: -- **Before the tag is pushed:** fix the problem and start a new run. -- **`build` fails after the tag push** (for example in *Upload dist artifact*): do not use - **Re-run failed jobs**, because a rerun of `build` stops at *Guard against duplicate tag*. - Confirm that nothing was published (neither index has `0.2.0` and no publish job ran), - delete the unused tag (`git push origin :refs/tags/v0.2.0`), and start a new run. -- **A publish job fails:** use **Re-run failed jobs** on the same run. - -A release candidate is not recommended here. Everything up to the tag push fails safely, -TestPyPI stages the upload, and an `rc` run would also publish the candidate to PyPI and run -`linear-complete`, closing the Linear release early. - -### 8. Verify the release - -On the publish path (step 7): - -- [ ] `git fetch --tags && git rev-parse v0.2.0^{commit}` equals the `main` commit you - released. -- [ ] exists, shows the README, lists - `Requires: Python >=3.11.0` and the MIT license, links to - `metamorphic-labs/orcapod-python`, and has both a wheel and an sdist. -- [ ] Install `orcapod==0.2.0` from PyPI in a clean environment for each Python version and - run the same pipeline as step 4. Nothing is checked out or built here. The install - ignores uv configuration files and index, find-links, override and constraint environment - variables and asks - PyPI only, and Python runs isolated (`-I`, no `PYTHONPATH`), so the test exercises the - PyPI artifact. `__version__` must print `0.2.0`: - - ```bash - ( - set -euo pipefail - work=$(mktemp -d) - for v in 3.11 3.12 3.13; do - env_dir="$work/pypi-$v" - uv venv --python "$v" "$env_dir" - py="$env_dir/bin/python" - env -u UV_INDEX -u UV_INDEX_URL -u UV_DEFAULT_INDEX -u UV_EXTRA_INDEX_URL -u UV_FIND_LINKS \ - -u UV_OVERRIDE -u UV_CONSTRAINT \ - uv pip install --python "$py" --no-cache --no-config \ - --default-index https://pypi.org/simple orcapod==0.2.0 - uv pip check --python "$py" - (cd "$work" && env -u PYTHONPATH "$py" -I -c " - import orcapod - from orcapod import function_pod - from orcapod.sources import DictSource - assert orcapod.__version__ == '0.2.0', orcapod.__version__ - src = DictSource(data=[{'name': 'Alice', 'age': 30}], tag_columns=['name'], source_id='people') - @function_pod(output_keys='birth_year') - def f(age: int) -> int: - return 2026 - age - rows = [(t.as_dict(), d.as_dict()) for t, d in f.pod(src).iter_data()] - assert rows == [({'name': 'Alice'}, {'birth_year': 1996})], rows - print(orcapod.__version__, rows) - ") - env -u PYTHONPATH "$env_dir/bin/orcapod" --help >/dev/null - done - echo "PYPI SMOKE OK in $work" - ) - ``` -- [ ] exists, has the - two dist files attached, and is marked Latest. Paste the `[0.2.0]` section of - `CHANGELOG.md` at the top of the release body — the generated notes are only a list - of PR titles and do not carry the migration instructions. -- [ ] `linear-sync` and `linear-complete` succeeded, and the merged issues moved to Done. - -On the tag-only path (chosen for 0.2.0): - -- [ ] `git rev-parse v0.2.0^{commit}` equals the release commit (the merge of this PR or a - later commit that passed step 4), and nothing for 0.2.0 appears on PyPI, TestPyPI or - GitHub Releases. - -### 9. Record and notify - -- [ ] On ITL-810: tick the last checklist item, add the tag and, if published, the PyPI and - GitHub Release links, then close it. (This PR says `Part of ITL-810`, so merging it does - not close the issue.) -- [ ] Tell anyone with an existing `0.1.x` pipeline database that they must migrate or opt - out before upgrading, and point them at *Breaking changes and migration*: the - pipeline-table migration fails on a store written by a released `0.1.x` (ITL-863), so - such nodes use `ignore_schema=("v0",)` and recompute; and pipeline tables written - before ITL-845 hold wrong source-info tokens until dropped (ITL-853). -- [ ] Tell the maintainers of `orcapod-extension-spikeinterface`: 0.2.0 changes stored - hash column types and function-pod signature hashes, so the extension should be - tested against it. -- [ ] Announce in the team's usual release channel. This checklist does not name people - or channels because none are recorded in the repository; the developer fills that in. - -### 10. Follow-ups after the release (not blockers) - -- [ ] **Upgrade the lockfile in a dedicated PR** right after publishing (ITL-822). Pinned - versions stay unchanged in the release itself. -- ITL-819 — restore S3 integration test coverage. -- SpiralDB integration tests fail on `main` with `403 Permission table:view denied`; - the test project's credentials need attention. -- The items under "Findings to decide on" in Part 3. - ---- - -## Part 3 — ITL-810 cut checklist and evidence - -### ☐ Freeze and record the selected release contents - -**Contents on `main` at `c5ab1e05`, plus this PR** — every other selected PR is merged; the -release freezes when #269 has merged. - -- Already on `main`: the 30 PRs in `git log --oneline v0.1.2..07e5a2ea`, each cited by - number in `CHANGELOG.md` `[0.2.0]`. -- Merged since: ITL-826 (#276, with ITL-825 #275), ITL-816 (#270), docs from PLT-4816 - (#271), ITL-817 (#272), ITL-827 (#277), ITL-818 (#266), ITL-834 (#279), ITL-813 + - ITL-832 (#278; #268 closed as superseded), ITL-849 (#281), ITL-845 (#282), ITL-837 with - ITL-854, ITL-857 and ITL-859 (#280), the MIT license (ITL-875, #283), the golden hash - suite (ITL-626, #284). -- Still to merge: this PR (#269, ITL-810). **Its merge is the last precondition of the - cut.** -- Explicitly excluded: #248 (ITL-604, moved to v0.3), #258, #145. - -### ☐ Record any explicitly approved release blockers - -| Blocker | Status | -|---|---| -| ITL-134 (#267) — `pygraphviz` breaks CI and the Release pre-flight | Merged 2026-11-09 (`07e5a2ea`) | -| ITL-813 — `MergeJoin` on empty input | Merged 2026-11-10 (#278, `ab544982`; #268 closed as superseded) | -| ITL-816 — `iter_data` drops rows on mismatched chunk layouts | Merged 2026-11-10 (#270, `cb3eed9a`) | -| ITL-817 — system tags misaligned across runs after a join | Merged 2026-11-10 (#272, `365a5f74`), as *part of* ITL-817: deterministic column order and name-aligned buffer materialization; positional default record IDs documented | -| ITL-826 — organization's required CI (with ITL-825, Python 3.13) | Merged 2026-11-10 (#276, `7dec4b54`) | -| ITL-832 — operator outputs keep the inputs' Arrow types | Merged 2026-11-10 (#278, `ab544982`) | -| ITL-834 — `Index` / `Pick` keep the input Arrow types | Merged 2026-11-10 (#279, `fde06d20`) | -| ITL-837 — `Join`'s asynchronous path keeps Arrow types | Merged 2026-11-10 (#280, `c438d4d1`), with ITL-854, ITL-857 and ITL-859 | -| ITL-845 — materializing rows stamps row 0's source info on every row | Merged 2026-11-10 (#282, `f18339e3`) | -| ITL-849 — `GroupBy` after `MergeJoin` raises `ArrowTypeError` | Merged 2026-11-10 (#281, `62a182d7`) | -| ITL-827 — packaging cleanup | Merged 2026-11-10 (#277, `8b92e608`); its proprietary license is replaced by #283 | -| ITL-875 — MIT license from 0.2.0 | Merged 2026-11-10 (#283, `4edfde8e`). The rest of ITL-875 (trusted publishers, rehearsal, public repository) decided only whether 0.2.0 is published (step 6: tag-only), not whether it is cut | - -Per the issue, ITL-626, ITL-620 and ITL-623 are not blockers. ITL-626's golden suite merged -anyway (#284, `c5ab1e05`) and the developer took it into 0.2.0. - -### ☐ Complete the existing release checks for the selected contents - -**Run on `61d0f18e`; must be repeated on the final `main` (step 4).** Environment: macOS -arm64, uv 0.13.7, no Graphviz, no Docker, no PostgreSQL. `pygraphviz` excluded with -`--no-install-package pygraphviz` on the coordinator's instruction. - -| Check | Command | Result | -|---|---|---| -| Lockfile current | `uv lock --check` | Pass | -| Tests, Python 3.11.18 | `uv sync --locked --all-extras --dev --no-install-package pygraphviz --python 3.11` then `uv run --no-sync --python 3.11 pytest -m "not postgres" --tb=short -q` | 4820 passed, **3 failed**, 16 skipped, 77 deselected, 2 xfailed | -| Tests, Python 3.12.15 | same with `3.12` | 4820 passed, **3 failed**, 16 skipped, 77 deselected, 2 xfailed | -| License check | `uv sync --locked --dev --no-install-package pygraphviz` then the `pip-licenses --allow-only=… --partial-match` command from `_license-check.yml` | Pass (exit 0) | -| Build | `uv build` | Pass — wheel and sdist built; as-is version `0.1.3.dev29+g61d0f18ee`, and `0.2.0` with `SETUPTOOLS_SCM_PRETEND_VERSION=0.2.0` standing in for the release tag | -| Wheel smoke test, 3.11 and 3.12 | clean venv, wheel installed with `pygraphviz` overridden out; import, run a source → function pod pipeline, `orcapod --help`, `orcapod migrate --help` | Pass on both | -| Full suite | `uv run --no-sync pytest tests/` (Python 3.12) | 4820 passed, **3 failed** (the same three), 93 skipped, 2 xfailed. The 93 skips are the 16 below plus the 77 PostgreSQL tests, which are opt-in (`--postgres`) | - -The 3 failures are identical on both versions and all need the Graphviz `dot` -executable, which this machine does not have (`FileNotFoundError: 'dot'`). CI installs it: - -- `tests/test_pipeline/test_graph_rendering.py::TestRenderGraphFullPath::test_render_with_output_path` -- `tests/test_pipeline/test_graph_rendering.py::TestRenderGraphFullPath::test_render_with_show_mocked` -- `tests/test_pipeline/test_graph_rendering.py::TestPipelineShowGraphFullPath::test_show_graph_with_show_mocked` - -The 16 skips: - -- 7 in `tests/test_databases/test_delta_table_database_s3.py` — MinIO container could not - be started (no Docker here; skipped in CI too, ITL-819). -- 9 in `tests/test_databases/test_spiraldb_connector_integration.py` — need - `SPIRAL_INTEGRATION_TESTS=1` and credentials. - -The 77 deselected are the `postgres`-marked tests, which the Release workflow also -deselects. The 2 expected failures, both "`list[T]` / `dict[K, list[T]]` where -`T` is a logical type not yet supported (PLT-1732)": - -- `tests/test_core/function_pod/test_write_side_registration.py::test_pod_declaration_with_doubly_nested_input` -- `tests/test_core/function_pod/test_write_side_registration.py::test_pod_declaration_with_doubly_nested_output` - -Smoke-test output (identical on 3.12 apart from the interpreter version): - -``` -python 3.11.18 | dist version = 0.2.0 | orcapod._version.__version__ = 0.2.0 | has orcapod.__version__: False -pipeline rows: [('Alice', 1996), ('Bob', 2001), ('Charlie', 1991)] -NodeConfig: NodeConfig(is_result_ephemeral=None, ignore_schema=('v0',)) | PodConfig: PodConfig(max_concurrency=2) -pygraphviz absent: import orcapod and pipeline run work without it -SMOKE OK -``` - -Wheel metadata: `Requires-Python: >=3.11.0`; extras `all`, `postgresql`, `ray`, `redis`, -`spiraldb`; 33 `Requires-Dist` lines; console script `orcapod`; `py.typed` and the two -context JSON files are included. - -**Patch-branch parity: nothing to port.** `git diff --stat v0.1.2 v0.1.4` changes one -file, `src/orcapod/core/executors/ray.py`. `git diff v0.1.4 origin/main` on that file -shows only that `main` writes the same `if opts:` branch as a conditional expression. -`main` covers the empty-options path in `tests/test_core/test_regression_fixes.py`. - -### ☐ Prepare release notes, deliberate breaking changes and known limitations - -**Done.** Placeholders resolved 2026-11-10 (step 2); #280, #281 and #282 folded in, and the -public release route described, on 2026-11-10. `CHANGELOG.md` `[0.2.0]`: *Breaking changes -and migration*, *Added*, *Changed*, *Fixed*, *Known limitations*. - -### ☐ Cut/publish v0.2 and record its tag/version or release link - -**Not done — the developer's step** (Part 2). - ---- - -## Findings to decide on - -None of these was changed in this PR. Findings 1–4 are addressed by ITL-827 (#277, -merged 2026-11-10); finding 1 again by #283. - -1. **There is no `LICENSE` file.** `pyproject.toml` declares MIT, but neither the - repository nor the wheel contains the licence text. *Decided 2026-11-09:* proprietary; - #277 added the `LICENSE` file and declared `LicenseRef-Metamorphic-Proprietary`. - *Reversed 2026-11-10:* MIT from 0.2.0; #283 (ITL-875) replaces the text and declares - `license = "MIT"`. -2. **The sdist publishes internal material.** It contains `superpowers/` (183 files of - agent plans and specs), `tests/`, `CLAUDE.md`, `DESIGN_ISSUES.md`, `plan.md`, - `notebooks/` and more — 3.3 MB against a 0.6 MB wheel. Earlier releases did the same. - #277 limits the sdist to the package source and its metadata files. -3. **`basedpyright` is a runtime dependency.** A type checker is installed for every - user. #277 moves it (and `typing_extensions`) to the `dev` group and removes the - unused `pymongo` and `pyyaml`. -4. **`orcapod.__version__` does not exist**, although `orcapod/_version.py` is generated. - #277 adds it. -5. **`v0.1.4` points at a commit on no branch.** `origin/patch` stops at `v0.1.3`. -6. **`CLAUDE.md` and `.zed/rules` still say "greenfield project pre-v0.1.0"** as the - reason not to keep backward compatibility. After 0.2.0 ships with a migration tool, - that policy deserves a deliberate restatement. -7. **`linear-sync` and `linear-complete` check out the default branch**, not the - `branch` input. Harmless for a release from `main`; wrong for a hotfix release. diff --git a/superpowers/plans/2026-11-09-workshop-facilitator-notes.md b/superpowers/plans/2026-11-09-workshop-facilitator-notes.md deleted file mode 100644 index 23c9079c8..000000000 --- a/superpowers/plans/2026-11-09-workshop-facilitator-notes.md +++ /dev/null @@ -1,175 +0,0 @@ -# Workshop Facilitator Notes (PLT-4864) - -Date: 2026-11-09, refreshed 2026-11-14 against `main` for v0.3.0. For the Orcapod workshop -with external collaborators on Thursday 2026-11-19. -Material: `docs/tutorial/` (parts 0-10), `docs/troubleshooting.md`. - -**Bottom line:** the tutorial is about 3.5 hours of self-paced work; a live session of 90 to -120 minutes should demo parts 1-6 and run four exercises live, and leave parts 7-10 as -self-study. Three decisions are needed from you before the day (first three open -questions). The two part-3 bugs of the first draft are fixed on `main`; the library -limits attendees can still meet are listed below. - -## Before the workshop: send this to attendees - -Ask them to do this the day before; it takes ten minutes and surfaces install problems -while there is still time. It is the content of `docs/tutorial/00-setup.md`. - -1. Python 3.11 or newer. -2. A fresh environment with Orcapod: - - ```bash - mkdir orcapod-workshop && cd orcapod-workshop - python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate - pip install orcapod - ``` - - (`uv init && uv add orcapod` works as well; then run every `python` command below as - `uv run python`.) -3. The dataset generator, in that directory: - - ```bash - curl -fO https://raw.githubusercontent.com/nauticalab/orcapod-python/main/docs/tutorial/workshop_data.py - python workshop_data.py - ``` - - Expected output: `Wrote 27 files (357 kB) to .../workshop_data`. -4. The check command: - - ```bash - python -c " - from workshop_data import make_dataset - from orcapod import PipelineJob, function_pod - from orcapod.databases import InMemoryArrowDatabase - from orcapod.sources import CSVSource - - @function_pod(output_keys='rate_hz') - def rate(n_spikes: int) -> float: - return n_spikes / 60 - - d = make_dataset() - job = PipelineJob(name='check', database=InMemoryArrowDatabase()) - with job: - units = CSVSource(str(d / 'units.csv'), tag_columns=['subject_id', 'session', 'probe', 'unit'], source_id='units') - rate.pod(units.select_data_columns(['n_spikes']), label='rate') - job.run() - print('OK', job.nodes['rate'].as_table().num_rows, 'rows') - " - ``` - - Expected last line: `OK 64 rows`. Anything else: reply with the full output. - -Optional: Graphviz (`dot` on the `PATH`), only for the one `show_graph()` sample in part 8. - -Tell them in advance that `Protobuf gencode version` warnings and -`WARN deltalake_core::writer::stats` lines are harmless. - -## Suggested live path, 105 minutes - -Times are for a room where everyone has completed the checklist. Demo by typing, not by -showing slides; attendees follow on their own laptops with the page open. - -| Min | Part | Do live | Skip or mention | -|---|---|---|---| -| 0-5 | Framing | One slide or two sentences: the goal is the per-session mean firing rate, stored and re-runnable. Show the dataset tree from the tutorial front page. | | -| 5-20 | 1. First stream | `CSVSource` for subjects and sessions; tags versus data; `output_schema`, `as_pandas_df`. **Exercise 1 live** (choose tags for `units.csv`, 4 min), then discuss the "is `region` a tag?" point. | `DictSource`: mention. Exercise 2 (the two errors): show both yourself in one minute. | -| 20-40 | 2. Function pods | `to_mm`, then `to_mm.pod(units)`: the pod drops the columns it does not take. `firing_rate` over two streams. **Exercise 1 live** (fix the failing call: a missing column, 3 min). | Multiple outputs: show `describe_depth`, no exercise. | -| 40-50 | 3. Operators | `join` on shared tags; `polars_filter` with an expression; `semi_join`. | `MergeJoin`, `map_data`: mention only. No live exercise. | -| 50-62 | 4. Many rows into one | `group_by` (it keeps only the `by` tags), the list-valued row, `mean_rate`. The result table with knockouts lower. | `batch`: one sentence. Exercise 1 (per probe) as a 3-minute warm-up if the room is fast. | -| 62-80 | 5. Keeping results | The `with` block, `run()`, `calls` going 0 → 71 → 71. Then `DeltaTableDatabase` and the second job that computes nothing. **Exercise 1 live** (`run_pipeline.py` twice, 5 min): this is the moment the product makes sense. | Directory layout: show `ls`, do not explain. | -| 80-97 | 6. Re-runs | Add two rows (2 calls). Then **edit the function and get the old numbers**: let it land, ask the room why. Then `version="v1.0"`. **Exercise 1 live** as a show of hands (predict a-d), then run it. | The hashes section: one sentence, point to the concept page. | -| 97-105 | Wrap-up | Open `docs/troubleshooting.md` and scroll it. Point to parts 7-10 by need: files (7) for anyone with per-session folders, tracing (8), failures (9). | | - -For a 90-minute slot: drop the part 4 warm-up and do part 3 in five minutes (join and -equality filter only). For 120 minutes: add part 7 up to "What makes a file result -recompute" (15 min); it is the most persuasive demo for people whose data is files. - -### Things that will come up, and the answer - -- **"I changed my function and nothing happened."** Part 6. Major version. This will - happen to someone during the session whether or not you have reached part 6; when it does, - use it. -- **"It says my schema is not compatible."** Part 2, rule 2. The text after the colon names - the column: missing, or a different type (an `int` column for a `float` parameter is - refused; Orcapod does not convert). -- **"Why 63 calls for 64 rows?"** Identical inputs share one result. It is in part 5; it is - a good question to take from the room. -- **"My output has the rows in a different order."** Rows read back from a database are not - ordered. Sort. (Part 5 says so.) -- **A wall of `WARN deltalake_core` lines** on the first Delta run. Harmless; warn people - before you press enter, because it looks like a failure on a projector. -- **The warning during part 9.** Under the default error policy a failed row is skipped - and the run carries on; it ends with one `RowFailuresWarning` and a run summary that - counts the failures. Say "the run finished; three rows failed" before running it. -- **"My node is empty."** A job node shows its current run: it is empty until `job.run()` - in this Python session, even when the database holds results. `node.as_source()` reads - everything stored. Parts 5 and 8. - -### What is deliberately not in the live path - -Side-effect pods, `Pipeline` blueprints and `bind`, save/load, observers, Ray. They are in -part 10 and `docs/concepts/pipeline-job.md`. Each needs explanation that does not pay off -in a first session. - -## State of the material - -- Every `python` sample in the tutorial, the two new concept pages and the troubleshooting - page runs in the test suite (`tests/test_docs/`), one skipped (`show_graph()`, needs - Graphviz). -- The tutorial's central claim, "a second run computes nothing", is pinned by - `tests/test_docs/test_rerun_computes_nothing.py` under five hash seeds (fixed by #272, - on `main`). -- The dataset generator is tested for byte-identical output - (`tests/test_docs/test_workshop_data.py`). -- Not tested: Windows; a real `pip install orcapod` of the released v0.3.0 wheel (the - samples ran against the source tree of this branch). - -## Library problems attendees will see - -Described as-is in the tutorial and on the troubleshooting page; none is worked around -silently. - -| Problem | Where it shows | What the tutorial says | -|---|---|---| -| `CSVSource` given a `pathlib.Path` and no `source_id` raises `TypeError` | Part 1 onwards | Wrap in `str(...)` | -| Editing a function body does not invalidate results | Part 6 | Bump the major version | -| `list(job.nodes)` order varies with the hash seed | Part 5 | Samples print `sorted(job.nodes)` | -| `job.show_graph()` does not exist; it is `job.pipeline.show_graph()` | Part 8 | Stated | -| The stored result of a node downstream of a join cannot be bound as a source of a new `PipelineJob` (`SourceSpecMismatchError`; ITL-1127) | Part 8 | The next stage runs in memory; to store it, add it to the first job | -| A `@sink_pod` function without a `ctx` parameter fails at run time | Part 10 | Stated | -| A sink does not drop data columns it has no parameter for; it fails at run time | Part 10 | Stated | -| A loaded job runs only with `run(load_only=True)`; its nodes are empty until then | Part 10 | Stated | -| Under the default error policy a failed row shrinks a downstream group | Part 9, exercise 3 | Read the run summary; `fail_fast` | -| Default record IDs are positional (ITL-817, ST2): reordering the rows of a source file changes row identity, so sinks re-deliver | Not shown; could surface if an attendee sorts a CSV | Not covered; `record_id_column` is documented in `docs/concepts/sources.md` | - -## Open questions for the developer - -1. **Is `pip install orcapod` the v0.3.0 release by workshop day?** The setup page says - `pip install orcapod` / `uv add orcapod`, with the GitHub install as the fallback. If the - release is not on PyPI in time, the checklist above must say - `pip install git+https://github.com/nauticalab/orcapod-python.git` instead (and - attendees then need `git`). `metamorphic-labs` is the private development repository; - attendee-facing URLs point at the public `nauticalab` one. -2. **Where will attendees read the tutorial?** The pages assume a published docs site, from - which `workshop_data.py` can be downloaded (MkDocs serves it next to the pages). If the - site is not published by then, the `curl` line from the repository works as long as the - repository is public under `metamorphic-labs`; otherwise hand out the file. Is the - repository public, and has the move from `nauticalab` happened? -3. **Should the generator ship in the package?** It lives in `docs/tutorial/workshop_data.py` - so that this PR touches nothing under `src/`. Shipping it as, say, - `python -m orcapod.tutorial_data` would remove the download step and the one command - attendees are most likely to get wrong. It needs no dependency. Your call for v0.2 or - later. -4. **The two part-3 bugs** (expression filter, `semi_join` key types) were fixed on `main` - (ITL-828 #298, ITL-829/847 #291); the warning boxes are gone. -5. **Does #267 (no `pygraphviz`) make v0.2?** The setup and troubleshooting pages assume - it does: they say Graphviz is optional and that the `pygraphviz` build failure means an - older version. -6. **`README.md` and `mkdocs.yml` still say `nauticalab`** (`repo_url`, the install line in - `docs/index.md`, `pyproject.toml` homepage). New text in this PR uses `metamorphic-labs`. - The existing lines belong to the release PR (#269) and were left alone. -7. **Ray.** The stale Ray notebook was removed. Part 10 mentions the `orcapod[ray]` extra in - one sentence and nothing more. If collaborators are expected to use a cluster soon, that - needs its own tested page. -8. **Windows.** Nothing here was run on Windows. If any attendee is on Windows, someone - should run the checklist on one machine before the day. diff --git a/superpowers/plans/2026-11-09-workshop-tutorial.md b/superpowers/plans/2026-11-09-workshop-tutorial.md deleted file mode 100644 index d752053b9..000000000 --- a/superpowers/plans/2026-11-09-workshop-tutorial.md +++ /dev/null @@ -1,220 +0,0 @@ -# Self-Paced Workshop Tutorial Implementation Plan (PLT-4864) - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. - -**Goal:** Ship a self-paced, tested tutorial (parts 0-10) for next week's workshop, with one -synthetic dataset, the two prerequisite concept pages, a troubleshooting page, working -replacements for the stale hands-on material, and facilitator notes. - -**Architecture:** Every tutorial page is a Markdown file under `docs/tutorial/` whose `python` -fences run under the existing docs harness (`tests/test_docs/`). Each page starts by calling -the dataset generator, so a page is runnable on its own in an empty directory. The generator -is a single standard-library script, `docs/tutorial/workshop_data.py`: MkDocs copies it into -the built site (so attendees can download it), the harness puts `docs/tutorial` on -`PYTHONPATH`, and an attendee runs `python workshop_data.py`. - -**Tech Stack:** MkDocs Material (`pymdownx.details` for collapsed solutions), pytest, the -existing `_doc_samples` runner. No new dependencies. No changes under `src/`. - -Source of the outline: `superpowers/plans/2026-11-09-docs-gap-report.md`, last section. - ---- - -## Decisions already taken - -| Decision | Choice | Why | -|---|---|---| -| Generator location | `docs/tutorial/workshop_data.py` | Not in `src/` (gate). Under `docs/` it is published with the site and sits next to the pages that use it. Confirmed by the coordinator. | -| Generator dependencies | Standard library only (`csv`, `random.Random(seed)`, `pathlib`) | Byte-identical output on every machine; nothing to install. | -| Solutions | Collapsed `??? success "Solution"` blocks on the same page | The solution code is then executed by the harness too. | -| Part 3 library failures | Teach the spellings that work; state both limits as known bugs with exact error text | Coordinator answer, option A (see "Library misbehaviour"). | -| Repository owner in URLs | `metamorphic-labs/orcapod-python` | Coordinator notice: the repo is moving; never write `nauticalab` or `walkerlab`. | -| Stale notebook | Delete `notebooks/tutorials/01_introduction_to_orcapod.ipynb` | Pre-approved. `docs/tutorial/` replaces it. | -| Stale example | Rewrite `examples/save_and_load_pipelines.py`; run it from the test suite | Brief prefers a working, tested replacement. | -| Environment | `uv sync --no-install-package pygraphviz`, then `uv run --no-sync ...` | `pygraphviz` cannot build here (no Graphviz headers); #267 removes it. | - -## The dataset - -`make_dataset(root="workshop_data") -> Path` writes about 0.4 MB, 27 files: - -``` -workshop_data/ - subjects.csv subject_id, sex, age_weeks, genotype 4 rows - sessions.csv subject_id, session, date, duration_s 8 rows - units.csv subject_id, session, probe, unit, region, - depth_um, n_spikes 64 rows - sessions/_/ - notes.txt - spikes_imec0.csv, spikes_imec1.csv time_s, unit -``` - -Four mice (two wildtype, two knockout), two sessions each, two probes per session, four -units per probe. Fixed seed; knockouts fire about 30% less so that the per-session summary -shows something. - -## Behaviour verified by running code (base: #271 branch) - -These are the facts the pages are written from. Each was observed, not read. - -- `CSVSource(path, tag_columns=..., source_id=...)` needs `path` as `str`; a `pathlib.Path` - raises `TypeError: 'PosixPath' object is not iterable`. Default `source_id` is the path. -- CSV string columns are Arrow `string`; `DictSource` and anything that passed through - `polars_filter` are `large_string`. -- Function pods: exact-column rule, annotation rule, multiple outputs via `output_keys=[...]`. -- `stream.as_pandas_df()` exists on sources, pod streams and job nodes. -- `PipelineJob(name=..., database=db)`; `job.nodes` lists sources by `source_id`, operators as - `SelectDataColumns_1`, `Join`, `GroupBy`, pods by `label`. -- Two input rows with identical data values share one stored result (64 unit rows, 63 calls). -- `DeltaTableDatabase("dir")` layout: `dir////v0` and - `dir//_result//`. Each write prints many - `WARN deltalake_core::writer::stats` lines to stderr. -- Editing a body: stale result. `version="v1.0"`: recomputed. -- `job.save(path)` / `PipelineJob.load(path)`: a loaded job answers - `job.nodes[label].get_all_records()`; `as_table()` on a loaded function node and - `run()` both raise `RuntimeError` (read-only stubs). -- `show_graph()` is on `Pipeline` (`job.pipeline.show_graph()`), not on `PipelineJob`, and - needs the Graphviz `dot` binary (`ExecutableNotFound` otherwise). -- `Pipeline` blueprint: `PipelineJob.from_pipeline(p, database=db)` → `unbound_sources`, - `job.bind(sources={...})`, then `run()`. -- Files: `op.File(path)` validates existence, is hashed by content (edit → recompute; same - content under another path → cache hit). A `pathlib.Path` column is hashed by its path - string (edit → no recompute). `op.Directory(path, ignore=[...])`. `GroupBy` gives - `list[File]`, usable as a `list[op.File]` parameter. A pod may return `Path` or `op.File`. -- Meta columns appear in `node.get_all_records(columns={"meta": True})`, not in - `node.as_table(columns={"meta": True})`. -- A function that raises: in memory the exception propagates; in a job the default policy - `"continue"` drops the failed rows and logs a traceback to stderr; - `job.run(orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast"))` re-raises. - `StatusObserver` / `LoggingObserver` record per-row state, stdout and traceback. -- `@sink_pod` functions must take the context parameter (`ctx`), otherwise - `TypeError: ... got an unexpected keyword argument 'ctx'`. - -## Library misbehaviour (reported, not fixed, not hidden) - -1. `polars_filter()` cannot be hashed → fails before a join and inside any - `PipelineJob` (`TypeError: SemanticAwarePythonHasher (strict): no implementation ... for - type 'polars.expr.expr.Expr'`). Equality constraints (`polars_filter(genotype="wildtype")`) - work everywhere. -2. `semi_join` between a `CSVSource` and a `DictSource`/filtered stream → - `ArrowInvalid: Incompatible data types for corresponding join field keys ... string and - ... large_string`. `join` on the same pair works. -3. `CSVSource` rejects a `pathlib.Path`. -4. `PipelineJob` has no `show_graph`; the gap report's outline assumed it. -5. A `sink_pod` without a `ctx` parameter fails at run time, not at decoration time. - -## File structure - -| File | Responsibility | -|---|---| -| `docs/tutorial/workshop_data.py` | Dataset generator and its CLI | -| `docs/tutorial/index.md` | What the tutorial is, the dataset, the list of parts | -| `docs/tutorial/00-setup.md` … `10-where-next.md` | Parts 0-10, one page each | -| `docs/concepts/pipeline-job.md` | `Pipeline` vs `PipelineJob`, stores, nodes, bind, save/load | -| `docs/concepts/file-types.md` | `op.File`, `op.Directory`, `Path`, what is hashed | -| `docs/troubleshooting.md` | Symptom → cause → fix, from the gap report table plus findings | -| `mkdocs.yml` | `Tutorial` nav section, two concept pages, troubleshooting | -| `tests/test_docs/test_doc_samples.py` | Page list + `docs/tutorial` on `PYTHONPATH` | -| `tests/test_docs/test_workshop_data.py` | Generator: deterministic, small, CLI works | -| `tests/test_docs/test_examples.py` | Runs `examples/save_and_load_pipelines.py` | -| `examples/save_and_load_pipelines.py` | Rewritten working example | -| `notebooks/tutorials/01_introduction_to_orcapod.ipynb` | Deleted | -| `superpowers/plans/2026-11-09-workshop-facilitator-notes.md` | Live path, checklist, open questions | -| `CLAUDE.md`, `.zed/rules` | "Documentation samples" section names the new tested paths | - -## Tasks - -### Task 1: Dataset generator (TDD) - -- [x] Write `tests/test_docs/test_workshop_data.py`: two runs into two directories give the - same relative file list and identical bytes; total size under 2 MB; `subjects.csv` has 4 - data rows, `sessions.csv` 8, `units.csv` 64; for each unit row, `n_spikes` equals the number - of rows for that unit in the matching `spikes_.csv`; - `python docs/tutorial/workshop_data.py out` exits 0 and prints `Wrote 27 files`. -- [x] Run it: `uv run --no-sync pytest tests/test_docs/test_workshop_data.py -q` → fails - (module does not exist). -- [x] Add `docs/tutorial/workshop_data.py` (from the scratch draft used for the probes). -- [x] Run again → passes. Commit `feat(docs): add deterministic workshop dataset generator`. - -### Task 2: Harness covers the new pages (TDD) - -- [x] In `test_doc_samples.py`, extend `test_doc_pages_are_discovered` with - `docs/tutorial/01-first-stream.md`, `docs/concepts/pipeline-job.md`, - `docs/troubleshooting.md`; run → fails. -- [x] Extend `DOC_PAGES` with `docs/tutorial/*.md` and `docs/troubleshooting.md`; add - `REPO_ROOT / "docs" / "tutorial"` to the subprocess `PYTHONPATH`. (The test stays red until - the pages exist; it goes green in Task 3.) - -### Task 3: Tutorial pages - -Common shape of every part: one-paragraph motivation; "You will" list; a first block that -calls `make_dataset()`; samples with shown output; "What to remember"; two or three exercises -with collapsed solutions; "Next" link. Terms are defined at first use. For each page: write -it, run `uv run --no-sync python -m tests.test_docs._doc_samples docs/tutorial/` with -the harness environment, correct the shown output from what was printed, re-run until green. - -- [x] `index.md`, `00-setup.md` (install, extras, check command, the warnings to ignore, get - the dataset). -- [x] `01-first-stream.md` (`CSVSource`, `DictSource`, tags vs data, `output_schema`, `keys`, - `iter_data`, `as_pandas_df`; exercises: choose tags for `units.csv`; provoke the two - construction errors). -- [x] `02-function-pods.md` (`firing_rate`; annotations; multiple outputs; exact-column rule - and `select_data_columns`; exercise: fix a pod that fails on a wide stream). -- [x] `03-operators.md` (`join`, `semi_join`, `map_data`, `polars_filter`, `Join` vs - `MergeJoin`; the two known bugs stated plainly; exercise: subjects + units, filtered). -- [x] `04-many-rows-into-one.md` (`group_by`, `batch`, pod over `list[float]`; exercise: - per-session mean from per-probe rows). -- [x] `05-pipeline-job.md` (record, run, `job.nodes`, in-memory vs Delta store; exercise: run - a script twice). -- [x] `06-reruns-and-caching.md` (new rows; edited body; major version; identity in brief; - exercise: which of four edits recompute). -- [x] `07-files.md` (`op.File`, `op.Directory`, `Path` outputs, file hash caching; exercise: - one output file per session). -- [x] `08-inspecting-and-tracing.md` (meta and provenance columns, `get_all_records`, - `as_source`, `show_graph`). -- [x] `09-when-things-go-wrong.md` (reading the common errors, error policy, status and - captured logs; links to the troubleshooting page). -- [x] `10-where-next.md` (sink pods, context injection, executors, `PollingSource`, - save/load; short samples only where verified). -- [x] Commit per two or three pages: `docs(tutorial): add parts N-M`. - -### Task 4: Concept pages and troubleshooting - -- [x] `docs/concepts/pipeline-job.md`, `docs/concepts/file-types.md`, - `docs/troubleshooting.md`; cross-link from `function-pods.md`, `file-hash-caching.md`, - `getting-started.md`, `index.md` where the old text pointed at a section. -- [x] `mkdocs.yml` nav. `uv run --no-sync pytest tests/test_docs/test_mkdocs_build.py -q`. -- [x] Commit `docs(concepts): add PipelineJob and file types pages, troubleshooting`. - -### Task 5: Stale hands-on material (TDD) - -- [x] `tests/test_docs/test_examples.py`: run `examples/save_and_load_pipelines.py` with - `cwd=tmp_path`, assert exit code 0 and that stdout contains `loaded job`. Run → fails with - the `AttributeError` from the gap report. -- [x] Rewrite the example (build, run, save, load, read stored records). Run → passes. -- [x] `git rm notebooks/tutorials/01_introduction_to_orcapod.ipynb`; grep the repo for - references to it. -- [x] Commit `fix(examples): replace broken save/load example; remove stale notebook`. - -### Task 6: Facilitator notes, agent instructions, DESIGN_ISSUES - -- [x] `superpowers/plans/2026-11-09-workshop-facilitator-notes.md`. -- [x] `CLAUDE.md` and `.zed/rules` "Documentation samples": add `docs/tutorial/`, - `docs/troubleshooting.md`, the `workshop_data` import, and the example test. -- [x] One appended `DESIGN_ISSUES.md` entry for this issue, listing the library findings. - -### Task 7: Verify and ship - -- [x] Time the harness: `uv run --no-sync pytest tests/test_docs -q --durations=20`; target - about a minute for the sample run. -- [x] `uv run --no-sync pytest tests/` — full suite; record pre-existing failures and skips by - name. -- [x] `git fetch`; if #271 merged, rebase onto `origin/main` and re-run. -- [x] Push, open the PR against `main` (`Fixes PLT-4864`, `Part of PLT-4816`), attach to - Linear, set the worktree PR, send `worker_done`. - -## Self-review - -- Brief items 1-7 map to Tasks 3, 1, 4, 4, 5, 2+7, 6. -- No dependency added; nothing under `src/`; `CHANGELOG.md`, `pyproject.toml` and README - install lines untouched. -- The plan changes no public API, hash value or stored schema. diff --git a/superpowers/plans/2026-11-10-itl-578-golden-merge.md b/superpowers/plans/2026-11-10-itl-578-golden-merge.md deleted file mode 100644 index 9daa40efe..000000000 --- a/superpowers/plans/2026-11-10-itl-578-golden-merge.md +++ /dev/null @@ -1,29 +0,0 @@ -# ITL-578 follow-up: merge main, move Polars pins into the golden suite - -**Goal:** bring PR #286 up to date with `main` (golden hash suite, ITL-626, merged in c5ab1e05), -prove the PR moves no pinned hash, and pin this PR's Polars hashes in the shared golden suite. - -## Steps - -1. Merge `origin/main` (no rebase). Resolve `DESIGN_ISSUES.md`: keep HS1 (main) and PL1 (this - PR), each once, PL1 last. -2. Run `tests/test_hashing/golden` unchanged: every pinned entry must pass, and rerunning the - generator must leave `data/` untouched. On any failure, stop and ask the coordinator. -3. Add Polars cases to `tests/test_hashing/golden/cases.py`, each a case of its own so that - running the generator against an older release (`--allow-errors`) loses only the Polars - entries: - - `semantic:polars_value.` — semantic hash of the five values pinned in - `test_polars_dataframe_hashing._GOLDEN` (fixtures moved verbatim). - - `datagram:polars_data_hash.` — content hash of the `Data` holding the value in - column `v` (the `_GOLDEN["data"]` construction, verbatim). - - `semantic:polars_type.{dataframe,series}`, `schema:polars_single_field.*` (stored Arrow type - and `Schema` hash, which name system-tag columns), and `datagram:polars_dataframe` / - `datagram:polars_series` stream summaries (same shape as main's pandas `dataframe` case). - TDD: `test_no_unpinned_entries` fails before the generator runs. -4. Run the generator. `git diff tests/test_hashing/golden/data` must show added lines only, and - the `polars_value` / `polars_data_hash` hex digests must equal the `_GOLDEN` values. -5. Remove `_GOLDEN`, `_golden_values` and `test_golden_hashes` from - `tests/test_hashing/test_polars_dataframe_hashing.py`; keep the property tests. Point the - docs and PL1 at the golden suite. -6. Full suite on 3.11/3.12/3.13, ruff check/format, basedpyright. Push, update the PR - description, address CI and CodeRabbit. diff --git a/superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md b/superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md deleted file mode 100644 index 50ab76a3b..000000000 --- a/superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md +++ /dev/null @@ -1,302 +0,0 @@ -# ITL-578 Polars DataFrame Logical Type Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Register `polars.DataFrame` as an Orcapod logical type so a function pod can take and -return a `pl.DataFrame`, and the value is stored, read back, hashed and cached. - -**Architecture:** A new `LogicalPolarsDataFrame` (extension name `polars.dataframe`, storage -`large_binary`) stores the frame as Arrow IPC stream bytes, the same envelope as -`LogicalPandasDataFrame`. The IPC helpers move from `pandas_type.py` to a shared -`logical_types/arrow_ipc.py` that both types import. The type is wired into the default data -context (`contexts/data/v0.1.json`) like pandas and numpy. Hashing goes through a semantic -handler whose design is **pending a developer decision** (Task 6). - -**Tech Stack:** polars 1.41.2, pyarrow 23.0.1, starfix 0.4.0, pytest. - ---- - -## Findings that shape the plan (investigation, November 10, 2026) - -Scratch scripts live in the session scratchpad; the numbers below come from them. - -1. **Storage round trip is exact** with `df.to_arrow(compat_level=pl.CompatLevel.oldest())`, - `combine_chunks()`, Arrow IPC stream, then `pl.from_arrow(...)`. Every dtype checked - returns with an identical Polars schema and `assert_frame_equal` passes: Int8–Int64, - UInt8–UInt64, Float16/32/64, String, Binary, Boolean, Categorical, Enum (category order kept, - via the `_PL_ENUM_VALUES2` field metadata), Decimal(p, s), Datetime (unit and time zone), - Date, Time, Duration (unit), List, Array (also 2-D), Struct (also with a nested Enum), Null, - an empty frame with a schema, and a zero-column frame. - `compat_level=oldest` is pinned so strings are written as `large_string`, never - `string_view`, whatever the Polars default becomes. -2. **Not round-trippable, so rejected at write time:** - - `Int128` / `UInt128`: `to_arrow` raises `ArrowInvalid: unsupported format string '_pli128'`. - - `Object`: `to_arrow` silently writes `fixed_size_binary[8]` (pointer bytes) and reads back - as `Binary`, which is silent corruption. - - Polars extension dtypes (`pl.BaseExtension`, e.g. an Orcapod `Extension('numpy.ndarray')` - column inside the frame): nested logical values inside a frame value are a separate design - question; reject now, report as a follow-up. - The check walks the dtype tree (List, Array, Struct children) so a nested `Object` is also - caught, and the error names the top-level column and the offending dtype. -3. **The IPC bytes are not canonical.** Equal frames give different bytes when the chunk layout - differs (fixable with `combine_chunks`) and when the bytes under null slots differ (not - fixable without rebuilding buffers). The bytes also carry Polars-version-specific field - metadata (`_PL_CATEGORICAL2`, `_PL_ENUM_VALUES2`). With no handler registered, the semantic - visitor hashes the stored bytes, so a handler is required. -4. **Starfix alone is not enough for a frame value.** `StarfixArrowHasher.hash_table` (what - `PandasDataFrameHandler` uses) is chunk-independent but, by starfix design, ignores column - order and struct field order and hashes a dictionary column as its value type - (Categorical == Enum(any categories) == String). It raises `NotImplementedError` for - `fixed_size_list` (Polars Array) and `null`. The existing pandas handler has the same - collisions on main (reported to the coordinator, out of scope here). - -The hash design (option A, B or C in the coordinator question of November 10, 2026) is held -for the developer. Task 6 is blocked until that answer arrives; everything else proceeds. - -## File structure - -| File | Responsibility | -|---|---| -| Create `src/orcapod/logical_types/arrow_ipc.py` | `table_to_ipc_bytes`, `ipc_bytes_to_table` (moved from `pandas_type.py`, unchanged bodies) | -| Modify `src/orcapod/logical_types/pandas_type.py` | Import the helpers from `arrow_ipc` | -| Create `src/orcapod/logical_types/polars_type.py` | `LogicalPolarsDataFrame`, `check_polars_schema_storable` | -| Modify `src/orcapod/logical_types/__init__.py` | Export `LogicalPolarsDataFrame` | -| Modify `src/orcapod/contexts/data/v0.1.json` | Register the logical type (+ handler after Task 6), changelog line | -| Modify `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | `PolarsDataFrameHandler` (Task 6, design pending) | -| Create `tests/test_logical_types/test_polars_type.py` | Unit tests: protocol, per-dtype round trip, rejections | -| Modify `tests/test_logical_types/test_roundtrips.py` | Builtin round trip over Parquet and Delta (mirrors pandas) | -| Create `tests/test_core/function_pod/test_polars_dataframe_values.py` | Public-interface integration tests | -| Create `tests/test_hashing/test_polars_handlers.py` | Hash properties (Task 6) | -| Modify `docs/concepts/function-pods.md` | "Value types" section listing built-in logical types incl. `pl.DataFrame` | -| Modify `DESIGN_ISSUES.md` | One appended entry for ITL-578 | - -No docs page lists pandas or numpy today (the brief assumed one). The plan adds a short -"Value types" section to `docs/concepts/function-pods.md` and reports this to the coordinator. - ---- - -### Task 1: Move the IPC helpers to a shared module (refactor, no behavior change) - -**Files:** Create `src/orcapod/logical_types/arrow_ipc.py`; modify `pandas_type.py`. - -- [ ] **Step 1:** Create `arrow_ipc.py` with `table_to_ipc_bytes(table: pa.Table) -> bytes` and - `ipc_bytes_to_table(data: Any) -> pa.Table`, bodies copied verbatim from - `pandas_type._table_to_ipc_bytes` / `_ipc_bytes_to_table`, Google docstrings. -- [ ] **Step 2:** In `pandas_type.py`, delete the two private functions and - `from orcapod.logical_types.arrow_ipc import ipc_bytes_to_table, table_to_ipc_bytes`; - update the four call sites. -- [ ] **Step 3:** Run `uv run pytest tests/test_logical_types/test_pandas_type.py tests/test_hashing/test_pandas_handlers.py -q` — expect all pass. -- [ ] **Step 4:** Commit `refactor(logical_types): share the Arrow IPC helpers between frame types`. - -### Task 2: Failing regression test for the decoration error - -**Files:** Create `tests/test_core/function_pod/test_polars_dataframe_values.py`. - -- [ ] **Step 1:** Write - -```python -import polars as pl -from polars.testing import assert_frame_equal - -from orcapod import function_pod # verify the public import path when writing - - -def test_pod_can_return_a_polars_dataframe(): - @function_pod(output_keys="df") - def make_frame(n: int) -> pl.DataFrame: - return pl.DataFrame({"i": list(range(n))}) - - assert make_frame is not None -``` - -- [ ] **Step 2:** Run it; expect `TypeError: No LogicalType or LogicalTypeFactory registered for `. - -### Task 3: `LogicalPolarsDataFrame` storage (TDD) - -**Files:** Create `src/orcapod/logical_types/polars_type.py`, -`tests/test_logical_types/test_polars_type.py`. - -- [ ] **Step 1: Tests first.** In `test_polars_type.py`: - - protocol: `isinstance(LogicalPolarsDataFrame(), LogicalTypeProtocol)`, - `logical_type_name == "polars.dataframe"`, `python_type is pl.DataFrame`, - Arrow extension name `polars.dataframe`, storage `pa.large_binary()`, Polars extension name. - - `python_to_storage` returns `bytes`. - - Parametrized round trip, one column per case, asserting - `back.schema == df.schema` **and** `assert_frame_equal(back, df)`: Int8 with null, UInt8, - UInt16, UInt32, UInt64, Float16, Float32, String with null, Binary with null, Boolean, - Categorical, Enum(["y","x","z"]), Decimal(10,2) and Decimal(38,2), Datetime("ms", - "America/New_York"), Datetime("ns"), Date, Time, Duration("ms"), List(Int64) with null, - List(String) with empty list, List(Categorical), Array(Int16, 2), Array(Int16, (1, 2)), - Struct({"a": Int64, "b": String}) with null, Struct({"e": Enum}), Null. - - A wide frame of all the above in one frame: column order and names preserved. - - Empty frame with schema (Int8 + Struct(Enum)): schema and zero height preserved. - - Zero-column frame `pl.DataFrame()`. - - Chunked frame (`pl.concat([...], rechunk=False)`, assert `n_chunks() == 2`) round trips. - - Rejections (`pytest.raises(ValueError, match=...)` matching column name and dtype): - `Object` column, `Int128`, `UInt128`, `List(Object)` nested, `Struct({"o": Object})` - nested, a Polars extension dtype column (`pl.Series([b"x"]).ext.to(LogicalNumpyArray().get_polars_extension_type())`). - - Non-DataFrame input (`pl.Series`, `pd.DataFrame`) raises `TypeError`. -- [ ] **Step 2:** Run `uv run pytest tests/test_logical_types/test_polars_type.py -q`; expect import failure. -- [ ] **Step 3: Implement** `polars_type.py`: - -```python -_UNSUPPORTED_LEAVES = (pl.Object, pl.Int128, pl.UInt128) - -def _find_unstorable_dtype(dtype: pl.DataType) -> pl.DataType | None: - """Return the first nested dtype that cannot round-trip, or None.""" - if isinstance(dtype, pl.BaseExtension): - return dtype - if isinstance(dtype, _UNSUPPORTED_LEAVES) or dtype in _UNSUPPORTED_LEAVES: - return dtype - if isinstance(dtype, (pl.List, pl.Array)): - return _find_unstorable_dtype(dtype.inner) - if isinstance(dtype, pl.Struct): - for field in dtype.fields: - found = _find_unstorable_dtype(field.dtype) - if found is not None: - return found - return None - -def check_polars_schema_storable(schema: pl.Schema) -> None: - """Raise ValueError naming the column and dtype that cannot round-trip exactly.""" - for name, dtype in schema.items(): - bad = _find_unstorable_dtype(dtype) - if bad is not None: - raise ValueError( - f"LogicalPolarsDataFrame: column {name!r} has dtype {dtype} " - f"(contains {bad}), which cannot be stored losslessly as Arrow IPC. " - "Supported: integer up to 64 bits, float, string, binary, boolean, " - "categorical, enum, decimal, date, time, datetime, duration, null, and " - "list/array/struct of those." - ) - -class LogicalPolarsDataFrame(BaseLogicalType): - # extension name "polars.dataframe": library-qualified like "pandas.dataframe" and - # "numpy.ndarray"; names the Python type users write, so stored schemas self-document. - ... - def python_to_storage(self, value, converter=None) -> bytes: - if not isinstance(value, pl.DataFrame): - raise TypeError(...) - check_polars_schema_storable(value.schema) - table = value.to_arrow(compat_level=pl.CompatLevel.oldest()).combine_chunks() - return table_to_ipc_bytes(table) - - def storage_to_python(self, storage_value, converter=None) -> pl.DataFrame: - frame = pl.from_arrow(ipc_bytes_to_table(storage_value)) - assert isinstance(frame, pl.DataFrame) # a Table always gives a DataFrame - return frame -``` - - (Extension class caching mirrors `LogicalPandasDataFrame` exactly.) -- [ ] **Step 4:** Run the unit tests; expect pass. -- [ ] **Step 5:** Commit `feat(logical_types): add polars.DataFrame logical type with Arrow IPC storage`. - -### Task 4: Register in the default data context - -**Files:** `contexts/data/v0.1.json`, `logical_types/__init__.py`. - -- [ ] **Step 1:** Before editing, record reference hashes with a scratch script (pipeline hash - and content hash of a small pod pipeline over `int`, `pd.DataFrame`, `np.ndarray` values, and - `get_default_context().semantic_hasher.hash_object(...)` of a few objects) to - `scratchpad/hashes_before.json`. -- [ ] **Step 2:** Add `{"_class": "orcapod.logical_types.polars_type.LogicalPolarsDataFrame", "_config": {}}` - after `LogicalPandasSeries`; append a changelog line; export from `__init__.py`. -- [ ] **Step 3:** Run Task 2's test; expect pass. Re-run the hash script; assert identical - to `hashes_before.json` (acceptance: existing hashes unchanged). Run - `uv run pytest tests/test_hashing tests/test_logical_types -q`. -- [ ] **Step 4:** Add the builtin round trip to `test_roundtrips.py` (Parquet + Delta), modelled - on `test_builtin_dataframe_round_trip`, with a frame holding Enum, Datetime(tz) and Struct. -- [ ] **Step 5:** Commit `feat(contexts): register polars.DataFrame in the default data context`. - -### Task 5: Public-interface integration tests (storage paths) - -**File:** `tests/test_core/function_pod/test_polars_dataframe_values.py`. -Every test drives pods/operators by calling them and reads results through -`iter_data()` / `as_table()` / node `get_all_records()`, then converts through the public -stream API. Frames are compared with `assert_frame_equal` **and** schema equality. - -- [ ] Pod output: `n: int -> pl.DataFrame` over a 3-row source; each emitted value is a - `pl.DataFrame` equal to the expected frame (Enum + Datetime(tz) + Struct + List columns). -- [ ] Pod input: source column of frames (built via `DictSource`/`ListSource` with an explicit - `pl.DataFrame` schema type) → `f(df: pl.DataFrame) -> int` returns `df.height`. -- [ ] Chained pods: produce → transform (`df.with_columns(...)`) → consume. -- [ ] `Batch` and `GroupBy` over a frame-valued data column: result column type is the logical - list type (`list[polars.dataframe]`), every member frame equal and in documented order. -- [ ] `Join` pass-through: frame column joined against another stream on the tag; frame unchanged. -- [ ] `PipelineJob` with `SyncPipelineOrchestrator` and `AsyncPipelineOrchestrator`, each over - `InMemoryArrowDatabase`, `DeltaTableDatabase(tmp_path)` and - `ConnectorArrowDatabase(SQLiteConnector(tmp_path / "db.sqlite"))`: records read back from the - node equal the produced frames. -- [ ] Cross-process cache hit: run a pipeline over a Delta store in a subprocess - (`sys.executable -c`, pattern from `tests/test_hashing/test_cross_process_stability.py`), - then in a second subprocess re-run it; the second run computes nothing (function call - counter via a marker file stays at the first-run count) and the frames read back are equal. - This test also needs Task 6 (the cache key is the hash). -- [ ] Commit `test(function_pod): polars.DataFrame values through pods, operators and databases`. - -### Task 6: Hash handler — decided by the developer (November 10, 2026) - -Decision: option A with corrections. Column order and struct field order are **not** part of -identity (starfix is deliberately order independent and Orcapod relies on it). Enum category -order **is**. The dtype descriptor is Orcapod's own versioned grammar, never Polars' repr or -`_PL_*` metadata. Values are hashed with starfix `hash_table` over a hash-only projection -(Array cast to `large_list`, top-level Null columns left out, row count in the descriptor). -`pl.Series` is approved as `polars.series` with the same scheme (name + dtype + values). The -pandas handler and every existing hash stay unchanged. - -Mechanism: a semantic handler per Python type (`PolarsDataFrameHandler`, -`PolarsSeriesHandler`) registered in `v0.1.json`. The column-level `ARROW:extension:*` -metadata route cannot carry a per-value dtype, so the handler route is the only one that -applies to every `polars.dataframe` value wherever it is hashed. The descriptor and hash -functions live in `polars_type.py`, next to the storage format they describe; the handlers -delegate to them. - -Grammar (canonical JSON, `sort_keys`, compact separators, UTF-8): -`{"v": 1, "kind": "polars.dataframe", "height": n, "columns": {name: dtype}}`, and for a -Series `{"v": 1, "kind": "polars.series", "name": s, "height": n, "dtype": dtype}`, where -`dtype` is one of `"int8"`…`"uint64"`, `"float16"`, `"float32"`, `"float64"`, `"boolean"`, -`"string"`, `"binary"`, `"date"`, `"time"`, `"null"`, `"categorical"`, -`{"enum": [categories in order]}`, `{"decimal": [precision, scale]}`, -`{"datetime": [unit, time_zone or null]}`, `{"duration": unit}`, `{"list": dtype}`, -`{"array": [dtype, size]}` (multi-dimensional arrays nest), `{"struct": {field: dtype}}`. -Anything else raises `ValueError` naming the column and dtype. Nested Null (inside List, -Array or Struct) cannot be left out without changing list offsets, so the projection casts it -to an all-null `bool` for hashing only; the descriptor still records `"null"`. -Digest: `sha256(descriptor_bytes + b"\x00" + starfix_digest)`, method `"sha256"`. - -Tests to write first: -`tests/test_hashing/test_polars_handlers.py` - -- equal frames hash identically regardless of chunk layout and of null-slot payload - (`[1, None, 3]` vs `[1, 2, 3]` nulled by `when/then`); -- differ in a value, a dtype (Int64 vs Int32; String vs Categorical vs Enum(a,b) vs - Enum(a,b,c)), a column name, column order, struct field order → different hashes; -- `Array` and `Null` columns hash without error; -- a frame read back from storage hashes the same as the original; -- cross-process stability: same hash in two interpreters with different `PYTHONHASHSEED`; -- a golden hex digest for one fixed frame, pinned in the test, to detect future drift. - -Then register `PolarsDataFrameHandler` in `v0.1.json` under `{"_type": "polars.DataFrame"}` -and in `register_builtin_python_type_handlers`; re-run the Task 4 hash comparison. - -### Task 7: Docs and design log - -- [ ] Add a "Value types" section to `docs/concepts/function-pods.md`: a table of the built-in - logical types (Python type, extension name, storage), including `polars.DataFrame`, the - rejected dtypes, and an executed sample of a pod returning a `pl.DataFrame` with its printed - output in `# ...` comments. -- [ ] `uv run pytest tests/test_docs -q` (doc samples + `mkdocs build --strict`). -- [ ] Append one DESIGN_ISSUES.md entry (ITL-578) recording the storage format, rejected - dtypes and the hash design. -- [ ] Commit `docs: document polars.DataFrame as a function pod value type`. - -### Task 8: Finish - -- [ ] `uv run --with ruff ruff format . && uv run --with ruff ruff check .`, - `uv run --with basedpyright basedpyright` (exit code 0), `uv run pytest tests/`. -- [ ] Merge `origin/main`, re-run the suite if it moved, push, open PR (`Fixes ITL-578`), - `orca linear attach`, `orca worktree set --worktree current --pr `, worker_done. - -Out of scope, reported in worker_done: `pl.LazyFrame`; `pl.Series` (held with the hash -decision); the pandas handler's column-order / dtype collisions; nested logical (extension) -dtypes inside a frame. diff --git a/superpowers/plans/2026-11-10-itl-578-polars-hash-labels.md b/superpowers/plans/2026-11-10-itl-578-polars-hash-labels.md deleted file mode 100644 index 4ccbf4d2d..000000000 --- a/superpowers/plans/2026-11-10-itl-578-polars-hash-labels.md +++ /dev/null @@ -1,86 +0,0 @@ -# Polars value hashes carry a versioned label (ITL-578, PR #286) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `PolarsDataFrameHandler` / `PolarsSeriesHandler` return `ContentHash` methods -`polars_dataframe_v1` / `polars_series_v1`, and the version lives only in that label. - -**Architecture:** Developer's decision (Nov 10, 2026): a per-type value hash carries a label -`_v` (as `ndarray_v1` in PR #289). The label names the whole definition (descriptor -grammar plus digest). The hashed descriptor loses its `"v"` key and -`POLARS_HASH_GRAMMAR_VERSION` goes away; the label constants replace it. Only the Polars golden -entries this PR added move (36 lines in `tests/test_hashing/golden/data`). - -**Tech Stack:** Python, polars, starfix, pytest, `uv`. - ---- - -### Task 1: Failing test for the labels - -**Files:** Test: `tests/test_hashing/test_polars_dataframe_hashing.py` - -- [ ] Add a test class that hashes a frame and a Series through the public semantic hasher - (`get_default_context().semantic_hasher.hash_object(value)`) and through a datagram - (`DictSource(...)` data `content_hash()` embeds the token) and asserts - `.method == "polars_dataframe_v1"` / `"polars_series_v1"`, and that the descriptor no longer - carries `"v"` (indirectly: label is the only version marker; a direct check of - `polars_dtype_descriptor` is unaffected because it never carried `"v"`). - -```python -class TestHashLabel: - """The method label names the hash definition and is its only version marker.""" - - def test_dataframe_label(self) -> None: - hasher = get_default_context().semantic_hasher - assert hasher.hash_object(pl.DataFrame({"a": [1]})).method == "polars_dataframe_v1" - - def test_series_label(self) -> None: - hasher = get_default_context().semantic_hasher - assert hasher.hash_object(pl.Series("a", [1])).method == "polars_series_v1" -``` - -- [ ] Run `uv run pytest tests/test_hashing/test_polars_dataframe_hashing.py -k Label -v`; - expect FAIL (`'sha256' != 'polars_dataframe_v1'`). - -### Task 2: Implement the labels - -**Files:** Modify `src/orcapod/logical_types/polars_type.py`, -`src/orcapod/hashing/semantic_hashing/builtin_handlers.py` (docstrings only), -`src/orcapod/contexts/data/v0.1.json` (changelog wording). - -- [ ] Replace `POLARS_HASH_GRAMMAR_VERSION = 1` with - `POLARS_DATAFRAME_HASH_METHOD = "polars_dataframe_v1"` and - `POLARS_SERIES_HASH_METHOD = "polars_series_v1"` (docstring: the label names the whole - definition; never change the bytes under a label, a new definition gets a new label). -- [ ] `_content_hash(descriptor, df, method)` returns `ContentHash(method=method, ...)`. -- [ ] Drop `"v"` from both descriptors; update the module docstring grammar and the - "versions the grammar" sentence; update "method sha256" in docstrings. -- [ ] Run Task 1 tests: PASS. Run the property tests: PASS (relative comparisons). - -### Task 3: Regenerate golden entries deliberately - -- [ ] `uv run python tests/test_hashing/golden/generate_golden_hashes.py` -- [ ] `git diff --stat tests/test_hashing/golden/data` and a script check: every changed line's - key starts with a `polars_` case and the count is 36 changed entries; no line added/removed - outside them; diff vs `origin/main` of `data/` remains additions only. -- [ ] Update the `cases.py` comment that names `POLARS_HASH_GRAMMAR_VERSION`. -- [ ] `uv run pytest tests/test_hashing -q`: PASS. - -### Task 4: Docs and instructions - -- [ ] `docs/reference/hashing.md` §1c: state the convention once, in general terms (label - `_v`, bytes under an existing label never change, a new definition gets a new label so - old and new hashes are told apart by inspection); fix the Polars formula/token example - (`polars:dataframe::polars_dataframe_v1:…`), drop `"v"` from the grammar block. -- [ ] `CLAUDE.md` and `.zed/rules`: one bullet in "Golden hash values" with the same rule. -- [ ] `DESIGN_ISSUES.md` PL1 entry: mention the labels. -- [ ] `uv run pytest tests/test_docs -q`. - -### Task 5: Verify, commit, push, PR - -- [ ] `git fetch`; merge origin/main if it moved. -- [ ] ruff check, ruff format --check, basedpyright (exit 0), full suite on 3.11/3.12/3.13. -- [ ] Commits: `feat(hashing): label Polars value hashes polars_dataframe_v1 / polars_series_v1 (ITL-578)` - and `docs(hashing): versioned per-type hash label convention (ITL-578)`. -- [ ] Push; update PR #286 description with an "Intended hash changes" section; wait for CI and - CodeRabbit; worker_done with head SHA. diff --git a/superpowers/plans/2026-11-10-itl-626-golden-generator-timeout.md b/superpowers/plans/2026-11-10-itl-626-golden-generator-timeout.md deleted file mode 100644 index 4682f9260..000000000 --- a/superpowers/plans/2026-11-10-itl-626-golden-generator-timeout.md +++ /dev/null @@ -1,77 +0,0 @@ -# Golden generator subprocess timeout Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** A hung golden hash generator fails the test with a clear message instead of -stalling the test run (CodeRabbit comment 4533438815 on PR #284, ITL-626). - -**Architecture:** `test_values_independent_of_process_and_hash_seed` runs -`generate_golden_hashes.py --stdout` in a subprocess with no timeout. Move that call into a -small helper, `_run_generator(env)`, that passes `timeout=600` and turns -`subprocess.TimeoutExpired` into `pytest.fail("golden generator timed out ...")`. No golden -value, case or generator code changes. - -**Tech Stack:** pytest, `subprocess`. - ---- - -### Task 1: Timeout on the generator subprocess - -**Files:** -- Modify: `tests/test_hashing/golden/test_golden_hashes.py` (the seed test near line 90) - -- [ ] **Step 1: Write the failing test** (append after the seed test) - -```python -def test_generator_timeout_fails_with_message(monkeypatch: pytest.MonkeyPatch) -> None: - # Exercises the private helper because the test is about its timeout handling. - def hang(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - raise subprocess.TimeoutExpired(cmd, kwargs["timeout"]) - - monkeypatch.setattr(subprocess, "run", hang) - with pytest.raises(pytest.fail.Exception, match="golden generator timed out"): - _run_generator(os.environ.copy()) -``` - -- [ ] **Step 2: Run it and confirm it fails** - -Run: `uv run pytest tests/test_hashing/golden/test_golden_hashes.py -k timeout -v` -Expected: FAIL, `NameError: name '_run_generator' is not defined`. - -- [ ] **Step 3: Implement** - -```python -GENERATOR_TIMEOUT_S = 600 - - -def _run_generator(env: dict[str, str]) -> subprocess.CompletedProcess[str]: - try: - return subprocess.run( - [sys.executable, str(GENERATOR), "--stdout"], - capture_output=True, - text=True, - env=env, - check=False, - timeout=GENERATOR_TIMEOUT_S, - ) - except subprocess.TimeoutExpired: - pytest.fail( - f"golden generator timed out after {GENERATOR_TIMEOUT_S}s " - f"(PYTHONHASHSEED={env.get('PYTHONHASHSEED')})" - ) -``` - -and in the seed test replace the inline `subprocess.run(...)` with -`result = _run_generator(env)`. - -- [ ] **Step 4: Verify** - -- `uv run pytest tests/test_hashing/golden/ -v`: all pass. -- `git diff --stat tests/test_hashing/golden/data`: empty. -- `uv run ruff format . && uv run ruff check .`, `uv run --with basedpyright basedpyright` - (exit code 0), `uv run pytest tests/`. - -- [ ] **Step 5: Commit, push, reply** - -`test(hashing): time out the golden generator subprocess (ITL-626)`; push; signed reply on -review thread 4533438815; wait for CI and the next CodeRabbit pass. diff --git a/superpowers/plans/2026-11-10-itl-626-golden-hash-regression.md b/superpowers/plans/2026-11-10-itl-626-golden-hash-regression.md deleted file mode 100644 index 4ae915647..000000000 --- a/superpowers/plans/2026-11-10-itl-626-golden-hash-regression.md +++ /dev/null @@ -1,166 +0,0 @@ -# ITL-626 Golden Hash Regression Suite Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Pin every hash value Orcapod persists or embeds in names (semantic hashes, schema -hashes, system-tag column names, Arrow table/schema hashes, per-row data hashes, record ids, -stream/node content and pipeline hashes, database table paths) in committed JSON golden files, -so any unintended change fails CI with a message naming the entry that moved. - -**Architecture:** One case catalogue module (`tests/test_hashing/golden/cases.py`) computes -every pinned value as a flat `{entry_name: json_value}` map per group. One generator script -(`tests/test_hashing/golden/generate_golden_hashes.py`) writes those maps to -`tests/test_hashing/golden/data/.json`. One test module -(`tests/test_hashing/golden/test_golden_hashes.py`) recomputes the maps and compares entry by -entry (one parametrized test per entry), checks the entry sets match, checks the invariants -the golden values encode (large == non-large, int8 != int64, nullable != non-nullable, -round-trip equality), and recomputes everything in subprocesses with different -`PYTHONHASHSEED` values. The same generator can be pointed at an older checkout -(`--output-dir`, `--allow-errors`) to produce the release-note diff against v0.1.4 and the -v0.2.0 candidate. - -**Tech Stack:** pytest, pyarrow, numpy, pandas, pydantic, orcapod public API -(`orcapod.sources`, `orcapod.operators`, `function_pod`, `PipelineJob`, -`orcapod.databases.InMemoryArrowDatabase`, `orcapod.contexts.get_default_context`). - ---- - -## Decisions - -- **Module path is part of identity.** Function pods, dataclasses and pydantic models hash - with their defining module. All of them live in `tests.test_hashing.golden.cases`; the - generator puts the project root on `sys.path` and imports that same dotted name, so pytest - and the script produce identical values. Never move these definitions. -- **Flat entries.** Each JSON file is a flat, sorted `{name: value}` object; names are dotted - (`pipeline.join_pod.pod.pipeline_hash`). Values are strings, or lists of strings for - column-name lists. Bytes are hex. A failure names the dotted entry. -- **Volatile by design, not pinned:** result-database record ids and the per-row token inside - function-pod output source info are UUIDv7 (time based); `__pod_ts`, git hash, Python - version columns. Pinned instead: the result DB's `__input_data_hash` (its lookup key) and the - pipeline DB's record ids, which are deterministic. -- **Known hasher gaps (ITL-848) are excluded**, not pinned as errors: `hash_schema` on a schema - with an extension type, `null`-typed columns, `arrow.uuid`, map and fixed-size-list values. - The generator header lists them so they are added when ITL-848 lands. -- **Lazy imports inside case builders** for anything absent from older releases (e.g. - `GroupBy`, `orcapod.logical_types`), so the generator can run against v0.1.4 with - `--allow-errors` and record `"ERROR: : "` for cases the old API cannot express. -- No change under `src/`. - -## File Structure - -- Create `tests/test_hashing/golden/__init__.py` — package marker. -- Create `tests/test_hashing/golden/cases.py` — case registry plus all cases, grouped: - - `semantic`: `BaseSemanticHasher` (default context) on scalars, collections, `Schema`, - type objects, UUID, `numpy.ndarray`, `pandas.DataFrame`. - - `schema`: `Schema` hashes over scalar / nested / optional / logical types (Path, UUID, - ndarray, DataFrame, dataclass, pydantic, `list[Path]`, `set[UUID]`); Arrow type string of - each Python type via the type converter; `compute_schema_hash` (full and truncated), - `compute_source_schema_hash`, `system_tag_column_names`; `SourceStreamBuilder.build()` - `schema_hash` / `system_tag_columns` / `table_hash` / record ids. - - `arrow`: `StarfixArrowHasher` `hash_table` / `hash_schema` over plain numeric, string, bool, - null-containing, nested list/struct, large vs non-large layouts, int8/16/32/64, nullable - vs non-nullable, column order, row order, `orcapod.path` / `orcapod.uuid` extension - columns, and an extension type with no registered handler (passthrough). - - `datagram`: public `DictSource` streams carrying each logical type — stream - `content_hash` / `pipeline_hash`, system-tag column names, per-row record ids and data - content hashes. - - `pipeline`: the end-to-end pipelines listed in the task, plus one `PipelineJob` on - `InMemoryArrowDatabase`. -- Create `tests/test_hashing/golden/generate_golden_hashes.py` — header documents when - regeneration is legitimate; CLI `--output-dir`, `--allow-errors`, `--group`, `--stdout`. -- Create `tests/test_hashing/golden/data/{semantic,schema,arrow,datagram,pipeline}.json`. -- Create `tests/test_hashing/golden/test_golden_hashes.py`. -- Modify `DESIGN_ISSUES.md` — append one ITL-626 entry. - -## Core code - -`cases.py` registry: - -```python -CaseFn = Callable[[], dict[str, Any]] -_CASES: dict[str, list[tuple[str, CaseFn]]] = {} - -def case(group: str, name: str) -> Callable[[CaseFn], CaseFn]: - def register(fn: CaseFn) -> CaseFn: - _CASES.setdefault(group, []).append((name, fn)) - return fn - return register - -def compute_group(group: str, *, allow_errors: bool = False) -> dict[str, Any]: - out: dict[str, Any] = {} - for name, fn in _CASES[group]: - try: - values = fn() - except Exception as exc: - if not allow_errors: - raise - out[name] = f"ERROR: {type(exc).__name__}: {exc}" - continue - for key, value in values.items(): - full = f"{name}.{key}" if key else name - assert full not in out, f"duplicate golden entry {full}" - out[full] = value - return dict(sorted(out.items())) -``` - -Stream summary helper used by `datagram` and `pipeline` cases (public interface only): - -```python -def stream_summary(stream) -> dict[str, Any]: - tag_keys, _ = stream.keys() - sys_cols = [k for k in stream.keys(system_tags=True)[0] if k not in tag_keys] - out = {"content_hash": stream.content_hash().to_string(), - "pipeline_hash": stream.pipeline_hash().to_string(), - "system_tag_columns": sys_cols} - for tag, data in stream.iter_data(): - row = _row_key(tag.as_dict()) - tag_all = tag.as_dict(all_info=True) - for col in sys_cols: - out[f"row[{row}].{col}"] = _jsonable(tag_all[col]) - out[f"row[{row}].data_hash"] = data.content_hash().to_string() - return out -``` - -Test module core: - -```python -@pytest.mark.parametrize(("group", "entry"), ALL_ENTRIES, ids=...) -def test_golden_entry(group, entry, live): - expected = GOLDEN[group][entry] - actual = live[group].get(entry, "") - assert actual == expected, ( - f"Golden hash entry {group}:{entry} moved.\n expected: {expected}\n" - f" actual: {actual}\nThis is a BREAKING change ... see generate_golden_hashes.py") -``` - -## Tasks - -### Task 1: Scaffolding and the failing test -- [ ] Create package, registry, generator, test module with one trivial `semantic` case and - no data file. Run `uv run pytest tests/test_hashing/golden -q`: FAILS (missing golden - file). Generate; rerun: PASS. Commit `test(hashing): golden hash regression scaffolding (ITL-626)`. - -### Task 2: semantic + schema + arrow groups -- [ ] Add the cases listed above, generate, run the suite and the invariant tests. Verify - `PYTHONHASHSEED=0` and `=12345` subprocess test passes. Commit. - -### Task 3: datagram + pipeline groups -- [ ] Add the logical-type streams and the pipelines: source→pod; source→Join→pod; MergeJoin; - SemiJoin; Batch; GroupBy; Index/Pick; MapData; Select/Drop columns; PolarsFilter with - keyword constraints; PipelineJob on `InMemoryArrowDatabase` (node hashes, `list_sources()` - table paths, pipeline-DB record ids, result-DB `__input_data_hash`). Generate; run twice - in separate processes and diff. Commit. - -### Task 4: Cross-interpreter check -- [ ] Run the generator with `--stdout` under Python 3.11, 3.12, 3.13 (separate - `UV_PROJECT_ENVIRONMENT`s) and diff against the committed files. Any difference → `ask` - the coordinator; do not pin per-version values. - -### Task 5: Release comparisons (report only, nothing committed) -- [ ] Scratch worktrees at PR #280 head (`e0f7fdb3`) and `v0.1.4`; run this generator with - `--allow-errors --output-dir ` against each; diff with the golden files; explain - each difference from the git history. - -### Task 6: Docs, DESIGN_ISSUES, full suite, PR -- [ ] Append the ITL-626 entry to `DESIGN_ISSUES.md`. Ruff, format, basedpyright, full - `uv run pytest tests/`. Merge origin/main, push, open PR (`Fixes ITL-626`), link, report. diff --git a/superpowers/plans/2026-11-10-itl-810-finalize-v0.2-release-docs.md b/superpowers/plans/2026-11-10-itl-810-finalize-v0.2-release-docs.md deleted file mode 100644 index 1f2c35df2..000000000 --- a/superpowers/plans/2026-11-10-itl-810-finalize-v0.2-release-docs.md +++ /dev/null @@ -1,93 +0,0 @@ -# ITL-810 — finalize the v0.2.0 release docs (PR #269) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this -> plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Every v0.2 PR is merged to `main` (`ab544982`). Bring PR #269 up to date with it -and replace every placeholder in the release docs with final wording and real PR numbers, -leaving the publish target as the one open item. - -**Architecture:** Documentation only. No source, workflow, dependency or lockfile change. -Each changelog entry is written from the merged PR description and checked against code -and tests on `main`. - -**Tech Stack:** Markdown, MkDocs (`mkdocs build --strict`), the docs sample harness -`tests/test_docs/`. - ---- - -### Task 1: Merge `origin/main` - -- [x] `git fetch origin && git merge origin/main --no-edit` (normal merge commit, no - rebase). Result: `abeae438`, textually clean. Git placed #278's own changelog entry - under the `[0.1.1]` *Breaking Changes* heading (main's `[Unreleased]` is this PR's - `[0.1.1]`); Task 3 moves it. - -### Task 2: Verify the facts against `main` - -Run small `uv run python -c` probes (scratchpad, not committed) for each claim: - -- [ ] ITL-817 (#272): `Join()(a, b)` and `Join()(b, a)` give identical `keys()`, schema and - `as_table()` columns; `MergeJoin` the same; `SemiJoin` keeps operand order. Find the - record-ID wording added to the docs (positional default, moved row = new entry, - function results reused). -- [ ] ITL-813/832 (#278): zero-row `MergeJoin`; `int8` merged column is - `large_list`; `PolarsFilter` output is large-normalized; disagreeing inputs - (`int8` vs `int64`) raise `InputValidationError`; `string` vs `large_string` merges; - record-ID system tags stay `fixed_size_binary[16]` through `PolarsFilter` and - `MergeJoin`; `Batch` over `int8` gives `large_list`; unregistered extension - element raises `ValueError`; `normalize_to_large_types` keeps nested non-null items. -- [ ] ITL-834 (#279): `Index` keeps `int8` pass-through columns and gives an `int8` - extracted column; extracting `None` where the predicted type is not optional raises - `ValueError`; `Datagram.with_columns` / `update` accept `pa.DataType` and `pa.Field`. -- [ ] ITL-827 (#277): `orcapod.__version__` exists; `pyproject.toml` license and deps. -- [ ] Known-limitation issue numbers (ITL-835, 837, 838, 839, 840, 841, 842, 843, 844, 819, - 822, 823): find each in `DESIGN_ISSUES.md`, code `TODO`s or Linear (read only). - -Anything that cannot be verified, or contradicts the coordinator's brief: `ask` and wait. - -### Task 3: `CHANGELOG.md` - -**Files:** Modify `CHANGELOG.md` - -- [ ] Move #278's entry out of `[0.1.1]` and replace the ITL-832 placeholder section in - `[0.2.0]` *Breaking changes and migration* with one section for ITL-813 + ITL-832 - (#278), including the stored-schema note for `list<...>` aggregates. -- [ ] Replace the ITL-834 placeholder section with the #279 entry. -- [ ] Add an ITL-817 (#272) entry (column order follows argument symmetry; no hash change; - positional record IDs). -- [ ] *Changed*: final lines for #278, #279, #277 (ITL-827). -- [ ] *Fixed*: final lines for ITL-813 (#278) and ITL-817 (#272). -- [ ] *Known limitations*: drop the `__version__` item (#277 merged); rewrite the ITL-835 - item; add ITL-837, 838, 839, 840, 841, 842, 843/844, 822/823. -- [ ] Keep the publish target open item in one place; update the RELEASE-DAY comment; - update the PR count line. - -### Task 4: `README.md`, `RELEASING.md`, `docs/` - -- [ ] One open-item statement per file; other mentions point to it. -- [ ] `git grep -n` for the capitalized placeholder marker finds nothing. - -### Task 5: Release checklist - -**Files:** Modify `superpowers/plans/2026-11-09-v0.2-release-checklist.md` - -- [ ] Record every PR as merged with its merge commit; #268 closed as superseded by #278. -- [ ] Step 2: placeholders resolved; publish target is the one open item. -- [ ] Step 3: answered from the merged PRs. -- [ ] Step 7 (trigger the workflow): **blocked** on the publish target open item. -- [ ] Part 3 blockers table and freeze list updated. - -### Task 6: Checks - -- [ ] `uv run pytest tests/test_docs/` (includes `mkdocs build --strict`) -- [ ] `uv run --with ruff ruff check .`, `uv run --with ruff ruff format --check .` -- [ ] `uv run --with basedpyright basedpyright` (exit code) -- [ ] `uv run pytest tests/` - -### Task 7: Commit, one push, CI, CodeRabbit, PR description - -- [ ] One `docs(release): …` commit; one push. -- [ ] Wait for CI green; CodeRabbit rounds (≤3, verify each finding, reply on each thread, - one push per round, poll ≥ 2 min apart). -- [ ] Update the PR description: contents, open item, head, CI. diff --git a/superpowers/plans/2026-11-10-itl-810-refresh-pr-269-public-release.md b/superpowers/plans/2026-11-10-itl-810-refresh-pr-269-public-release.md deleted file mode 100644 index 1ebd4b0e6..000000000 --- a/superpowers/plans/2026-11-10-itl-810-refresh-pr-269-public-release.md +++ /dev/null @@ -1,71 +0,0 @@ -# ITL-810 — refresh PR #269 for merged fixes and the public release route - -Coordinator task `task_d429e47e6104`, 2026-11-10. Docs-only change on -`eywalker/itl-810-cut-orcapod-python-v02-by-november-1` (PR #269). Refs ITL-875. - -## Starting state (checked before any change) - -- Branch head `9fe6725c`, equal to `origin/eywalker/itl-810-cut-orcapod-python-v02-by-november-1`. -- Only untracked file: `superpowers/plans/2026-11-09-pr-269-coderabbit-review.md` (the - CodeRabbit log, deliberately never committed). -- Merge base with `origin/main` is `ab544982` (#278), and #279 is already in it. `main` is 3 - commits ahead: #281 (ITL-849, `62a182d7`), #282 (ITL-845, `f18339e3`), #280 (ITL-837 with - ITL-854, ITL-857 and ITL-859, `c438d4d1`). - -## Constraints - -- No source, `LICENSE`, `pyproject.toml`, `tests/test_packaging.py` or `release.yml` edits. - The license change is ITL-875's own PR. -- Merge `origin/main` (no rebase, no force-push). The tag is the coordinator's job. -- Every changelog claim is checked against merged code (a snippet I run or a test I cite). - -## Steps - -1. **Merge `origin/main`.** Main's three new entries land under `[0.1.1]` in this branch, - because this branch renamed `[Unreleased]` to `[0.1.1]`. Move each into `[0.2.0]` so - it appears once: - - ITL-849 → *Fixed* (it adds a capability over already aggregated streams). - - ITL-845 → *Fixed*, with main's wording kept verbatim (developer's wording on stored - rows: not repaired, drop pipeline tables, keep result tables, ITL-853). - - ITL-837 (+ ITL-859) and ITL-854/ITL-857 → *Breaking changes and migration*, because - each changes behaviour or hashes on a documented path. - - Add the matching one-line *Changed*/*Fixed* index lines with PR numbers. -2. **Known limitations.** Update: - - ITL-835: `Join` now applies the disagreeing-types rule too (verify which operators). - - ITL-841: re-check which operators emit large variants after #280. - - Add: ITL-863 (migration crashes on a real v0.1.4 store; workaround - `ignore_schema=("v0",)` and recompute), ITL-853 (stored source-info tokens written before - ITL-845 stay wrong until pipeline tables are dropped), ITL-867 (meta columns have no - written rule), ITL-865 and ITL-856 (async path with zero-row inputs). ITL-851 is - already listed; keep it, sharpen the wording. - - Remove nothing that is still true. -3. **Public release route (developer decision 2026-11-10).** Rewrite every internal-registry, - not-on-PyPI and proprietary sentence in `CHANGELOG.md`, `README.md`, `docs/index.md`, - `RELEASING.md` and the release checklist: - - Install: `pip install orcapod` / `uv add orcapod`. - - License: MIT from 0.2.0 (applied by the ITL-875 PR; a cut precondition). - - Release: manual dispatch of `release.yml`, which runs `validate-branch`, tests on - 3.11–3.13 and the license check, `build` (creates and pushes the tag, builds wheel and - sdist), `publish-testpypi` then `publish-pypi` through OIDC trusted publishing (`uv - publish`, environments `testpypi`/`pypi`), creates the GitHub Release, and runs the - Linear jobs. Describe exactly that; drop the "(superseded)" markers. - - Open preconditions in the checklist: trusted publisher on PyPI and TestPyPI pointing at - `metamorphic-labs/orcapod-python` (only the PyPI project owner can do it); the license - PR merged; v0.2.0 is published only if the route is rehearsed in time, otherwise the tag - only marks the commit and v0.3.0 is the first public release from this repository. - - Point out the interaction between a hand-made tag and the workflow's duplicate-tag - guard (a hand-made `v0.2.0` tag makes a later `release.yml` run for `0.2.0` abort). -4. **Checklist refresh.** Rows 10–12 merged with merge commits; precondition boxes ticked - with evidence; blockers table; freeze list; Part 3 status; *Findings* 1 updated (MIT). -5. **Verify.** Run snippets for each new changelog claim (Join type error, Join meta drop, - Datagram.copy schema, per-row source info, GroupBy after MergeJoin, Index/Pick types, - nested nullability) on the merged tree, or cite the merged test. Then the four CI - commands and `tests/test_docs`. -6. **PR.** Push (no force), update the PR body: keep `Part of ITL-810`, add `Refs ITL-875`, - new status and test plan. Request a CodeRabbit round, address it, wait for CI green. - `orca linear attach`, `orca worktree set --pr 269`, then `worker_done`. - -## Out of scope - -- Fixing any of the listed known issues. -- `pyproject.toml` URLs already changed earlier in this PR stay as they are. diff --git a/superpowers/plans/2026-11-10-itl-813-832-mergejoin-arrow-types-rework.md b/superpowers/plans/2026-11-10-itl-813-832-mergejoin-arrow-types-rework.md deleted file mode 100644 index 211e00d0f..000000000 --- a/superpowers/plans/2026-11-10-itl-813-832-mergejoin-arrow-types-rework.md +++ /dev/null @@ -1,146 +0,0 @@ -# MergeJoin zero-row inputs + input Arrow types (ITL-813, ITL-832) — rework - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this -> plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make PR #278 the single PR for ITL-813 and ITL-832, reworked to the developer's -review of #268: no Arrow inference from Python values in `MergeJoin`, merged columns typed -`large_list` (or the logical list type of a logical element), and one -isolated rule for inputs whose Arrow types disagree. - -**Architecture:** `arrow_utils.aggregated_list_type` is the one list-type construction for -`MergeJoin`, `Batch` and `GroupBy`; it returns `pa.large_list(T)` for a plain element and the -converter's `list[]` extension type for a logical element, raising if the converter -cannot supply one. `MergeJoin` snapshots both input tables' Arrow types before the Polars -join, resolves every column present in both inputs through `_resolve_disagreeing_types` -(layout-only disagreement -> large variant, anything else -> `InputValidationError`), builds -merged columns with an explicit type and restores every pass-through column's type. - -**Tech Stack:** PyArrow 23, Polars, pytest, uv. - -**Decisions (developer review of #268, relayed by the coordinator; not re-asked):** -large variants, never non-large; reuse orcapod's type converter, no hand-written inference -table; a list of a logical type is the logical list type; inputs' Arrow types are respected. - ---- - -## Already done before this plan - -- Merged `origin/main` (365a5f74, PR #272 canonical column order) into the branch; resolved - `DESIGN_ISSUES.md` keeping every entry once. -- Merged the #268 branch (0b8a8bb2) for its last test commit `d0678dbc` (merged source-info - type parity); resolved `tests/test_core/operators/test_empty_inputs.py`. -- After those merges one pinned case of `TestDisagreeingArrowTypes` fails (main's canonical - join order changed what the old "pre-ITL-832" behaviour produces); that class is replaced in - Task 3 anyway. - -## File map - -- `src/orcapod/utils/arrow_utils.py` — `aggregated_list_type` (large_list, logical-list error), - `list_array` (unchanged contract: explicit type), `build_aggregated_table` (passes column name). -- `src/orcapod/core/operators/merge_join.py` — delete `_inferred_type`, `_merged_list_array`'s - inference branches; `_resolve_disagreeing_types` implements the rule; `_input_types` raises - through it; every output column type comes from the inputs. -- `tests/test_utils/test_arrow_utils_list_types.py` — helper unit tests. -- `tests/test_core/operators/test_merge_join_arrow_types.py` — element types, zero-row parity, - disagreeing types, hash pins. -- `tests/test_core/operators/test_empty_inputs.py` — expected merged type now `large_list`. -- New `tests/test_core/operators/test_aggregated_list_types.py` — Batch/GroupBy large_list, - logical list preserved, hash equality across list/large_list. -- Docs: `docs/concepts/operators.md`, `DESIGN_ISSUES.md` (MJ1, MJ2), `CLAUDE.md` and - `.zed/rules` (both). - -## Task 1: `aggregated_list_type` returns `large_list`; logical element must map to a logical list - -- [ ] Failing tests in `tests/test_utils/test_arrow_utils_list_types.py`: - - `aggregated_list_type(T, tc, column="c") == pa.large_list(T)` for int8, float32, string, - large_string, struct, list. - - Path element -> `tc.python_type_to_arrow_type(list[Path])`, an `ExtensionType` named - `list[orcapod.path]` with storage `large_list`. - - A foreign extension element (`pa.uuid()`; converter maps it to `Any` and returns a plain - `large_list`) raises `ValueError` naming the column and the type. -- [ ] Run, see failures (`list` != `large_list`, no error raised). -- [ ] Implement: - -```python -def aggregated_list_type(element_type, type_converter, *, column: str) -> pa.DataType: - if isinstance(element_type, pa.ExtensionType): - element_python_type = type_converter.arrow_type_to_python_type(element_type) - list_type = type_converter.python_type_to_arrow_type(list[element_python_type]) - if not isinstance(list_type, pa.ExtensionType): - raise ValueError( - f"Column {column!r}: cannot build a list of logical type {element_type}; " - f"the type converter has no logical list type for it (got {list_type})." - ) - return list_type - return pa.large_list(element_type) -``` - - `build_aggregated_table` passes `column=field.name`. -- [ ] Run helper tests; commit `fix(arrow_utils): build aggregated lists as large_list and require a logical list for logical elements (ITL-832)`. - -## Task 2: Batch / GroupBy go through the helper (large_list, logical list preserved, hashes unchanged) - -- [ ] New `tests/test_core/operators/test_aggregated_list_types.py`: - - Batch and GroupBy over an `int8`/`large_string`/struct data column: output column - `large_list`; source-info column `large_list`; prediction via - `unary_static_process(stream).output_schema(...)` equals `op.output_schema(stream)`. - - Path column: output type is the `list[orcapod.path]` extension, storage - `large_list`, values round-trip. - - Hash pin: each row's tag/data `content_hash` is identical to the same output cast to the - `list<...>` layout main produced (list/large_list hash the same). -- [ ] Run (fails on `large_list` assertions), confirm Task 1 makes them pass; fix any existing - test that pinned `list<...>` for Batch/GroupBy. -- [ ] Commit `test(operators): pin large_list and logical list types for Batch and GroupBy (ITL-832)`. - -## Task 3: MergeJoin — no inference, large_list merged columns, disagreement rule - -- [ ] Rewrite tests in `tests/test_core/operators/test_merge_join_arrow_types.py`: - - `TestMergedColumnElementType`: for every `ELEMENT_CASES` type and `rows in (0, 2)`, merged - type `== pa.large_list(T)`; merged source-info type `== large_list`; zero-row - left only, right only, both (struct and Path included) give the same types as non-empty, - match `static_process(...).output_schema(...)` vs `op.output_schema(...)`, and - `pa.concat_tables` of empty + non-empty works. - - Narrow never widens: `int8` input -> `large_list`, `float32` -> `large_list`. - - `TestDisagreeingArrowTypes` replaced: - - layout-only pairs (string/large_string, binary/large_binary, list/large_list, - struct/struct, list/large_list) for a - colliding column -> `large_list`, both argument orders, rows 0 and 2; for a - shared tag key -> the large variant. - - width/other pairs (int8/int64, float32/double, timestamp ms/us, decimal precisions, - struct with different fields) -> `InputValidationError` naming the column and both - types, for colliding column and shared tag, both orders, rows 0 and 2. - - CSV (`string`) vs dict (`large_string`) label column merges to `large_list`. - - Direct unit tests of `_resolve_disagreeing_types`. - - Hash pins: merged/pass-through rows hash equal to the `list` layout (layout-only change); - the width case: int8 merged column hashes differently from the int64-widened layout main - produced, while an int64 input hashes the same as main's layout. -- [ ] Run, see failures. -- [ ] Implement in `merge_join.py`: - - delete `_inferred_type`; replace `_merged_list_array` with a call to - `arrow_utils.list_array(values, arrow_utils.aggregated_list_type(t, tc, column=col))`; - - `_resolve_disagreeing_types(name, left, right) -> pa.DataType` returns the pointwise large - unification or raises `InputValidationError(f"MergeJoin: column {name!r} has Arrow type - {left} in one input and {right} in the other ...")`; - - `_input_types` always returns a type for every column (no `None`); - - schema reconstruction uses `output_types[col]` (input/resolved type for pass-through, - built list type for merged) — no fallback to whatever the join produced. -- [ ] Run tests; commit `fix(merge_join): type merged columns large_list and reject non-layout type disagreements (ITL-813, ITL-832)`. - -## Task 4: Existing tests and docs - -- [ ] `tests/test_core/operators/test_empty_inputs.py`: merged type `large_list`. -- [ ] grep tests for `pa.list_(` assertions on Batch/GroupBy/MergeJoin outputs and update. -- [ ] `docs/concepts/operators.md`: Arrow type rules for MergeJoin/Batch/GroupBy. -- [ ] `DESIGN_ISSUES.md` MJ1 and MJ2 fix notes (MJ2 -> resolved). -- [ ] `CLAUDE.md` and `.zed/rules`: aggregated list columns are `large_list`; logical list - rule; disagreement rule. -- [ ] Commit `docs: record large_list aggregated columns and the disagreement rule (ITL-813, ITL-832)`. - -## Task 5: Verification and PR - -- [ ] `grep -n "pa.array(\|from_pylist" src/orcapod/core/operators/merge_join.py` -> every hit - has an explicit type. -- [ ] ruff check, ruff format --check, basedpyright (exit code), pytest on 3.11/3.12/3.13, - `uv run pytest test-objective/`. -- [ ] fetch, merge main if moved, one push, retitle/rewrite PR #278, CI, CodeRabbit rounds. diff --git a/superpowers/plans/2026-11-10-itl-837-join-async-arrow-types.md b/superpowers/plans/2026-11-10-itl-837-join-async-arrow-types.md deleted file mode 100644 index 4ad7364a8..000000000 --- a/superpowers/plans/2026-11-10-itl-837-join-async-arrow-types.md +++ /dev/null @@ -1,150 +0,0 @@ -# ITL-837: Join's async path keeps Arrow types — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `Join.async_execute` yields rows with exactly the Arrow types, nullability, column -order, system tags and Tag/Data content hashes of `Join.static_process`, for two and for three -or more inputs; every other operator with its own async path is checked by the same test. - -**Architecture:** The async merge helpers (`_merge_pair_rename`, `_merge_pair_passthrough`, -`_rename_sys_tags`) stop rebuilding `Tag` / `Data` from `as_dict()` Python values. They build -the merged row from the incoming datagrams' Arrow tables (`as_table()`), so no type is ever -inferred. The data context comes from the incoming datagrams (left/canonical-first datagram's -`data_context`); the operator gains no state and no protocol changes. - -**Tech Stack:** Python, PyArrow, pytest, asyncio channels (`orcapod.channels.Channel`). - ---- - -## Reproduction (before the fix, on origin/main ab544982) - -Two `ArrowTableSource`s, tag `k: int16`, data `a: int8` / `b: int8`: - -``` -sync tag: k: int16 | data: a: int8, b: int8 -async tag: k: int64 not null | data: a: int64 not null, b: int64 not null -Tag/Data content hashes differ for every row. -``` - -## Sync semantics to mirror (read from `static_process`) - -* Column order: tag and data columns by first appearance across inputs in canonical - (`pipeline_hash`) order. The async path already joins in canonical order (PR #272): left is - canonically first, so "left columns, then right-only columns" reproduces it, also for the - staggered `join(join(x, y), z)` chain. -* Shared tag (join-key) columns: `restore_schema_nullability(table, left_ref, right_ref)` — - the last (right) reference wins, so a shared key takes the **right** input's field - (nullability). Values are equal by construction (they matched). -* Data columns: left fields from left, right fields from right; no overlap (Join validates). -* System tags: Python dicts on the `Tag` in both paths (sync `iter_data` builds `Tag` from the - table and extracts system tags via the converter), so passing the dict values through is - already equivalent. - -## Rules (developer, non-negotiable) - -* No Arrow type inference from Python values when a type is known; no bare `pa.array` / - `from_pylist` without a schema; converter only when nothing is known (not needed here). -* Large-normalized types; width, nullability, struct fields, logical types never change. - Disagreement is an error: a shared tag key whose large-normalized Arrow types differ between - the two sides raises `InputValidationError` in the async path. (Sync currently silently takes - the right side's type — a pre-existing sync defect; pinned with a strict xfail and reported, - not fixed here: changing it is a sync behaviour change.) -* Context from the datagrams, never `self.data_context` / `contexts.resolve_context`. - -## File Structure - -* Modify `src/orcapod/core/operators/join.py` — new static helpers `_merge_tag_tables`, - `_merge_data_tables`, `_build_tag`; rewrite `_merge_pair_rename`, `_merge_pair_passthrough`, - `_rename_sys_tags` to use them. -* Create `tests/test_core/operators/test_async_sync_equivalence.py` — the sync-vs-async - equivalence matrix over Join (2, 3 inputs) and every operator with an async path, plus a - pinned-hash test and the shared-key disagreement tests. -* Docs: `CHANGELOG.md` (one v0.2.0 entry), `DESIGN_ISSUES.md` (one entry appended at the - end), `docs/concepts/operators.md` only if it describes async behaviour. - -### Task 1: Failing equivalence test - -**Files:** Create `tests/test_core/operators/test_async_sync_equivalence.py` - -- [ ] Build three non-identical sources over shared tag `k: int16` (input A also has tag - `ta: int8`, input C tag `tc: uint8`), with data columns spread across them: `int8`, - `int16`, `int32`, `uint8`, `float32`, `string`, `large_string`, `binary`, `list`, - `large_list`, `list` (non-nullable items), a struct, a `Path` - logical column (via the default type converter), a nullable column holding a null and - non-nullable columns (cast `nullable=False`). -- [ ] `_sync_rows(op, *sources)` = `op.process(*sources).iter_data()`; - `_async_rows(op, *sources)` feeds each source's `iter_data()` into a `Channel`, runs - `op.async_execute(readers, out.writer, input_pipeline_hashes=[s.pipeline_hash() ...])` - and collects. -- [ ] `_assert_equivalent(sync_rows, async_rows)`: match rows by a stable key (repr of sorted - user-tag dict + sorted data dict); for each pair assert equal - `tag.arrow_schema(columns={"system_tags": True})`, `data.arrow_schema(columns={"source": True})` - (types AND nullability, field order), `list(tag.keys())`, `list(data.keys())`, - `tag.content_hash()`, `data.content_hash()`, `list(tag.system_tags().items())`, - `data.source_info()`, and the `arrow_hasher.hash_schema` of the combined row schema plus - the `semantic_hasher.hash_object` of `(tag.schema(), data.schema())`. -- [ ] Cases: Join(A,B), Join(B,A), Join(A,B,C), Join(C,A,B); MergeJoin on two inputs with - colliding non-struct columns; SemiJoin(A,B); Batch(2), Batch(0); GroupBy(["ta"]); - PolarsFilter(constraints); MapTags; MapData; Select/Drop Tag/Data columns; Index on the - `list` column; Pick on a `dict[str, int16]` column. -- [ ] Run `uv run pytest tests/test_core/operators/test_async_sync_equivalence.py -v`; - expected: the Join cases FAIL (int64 vs int16/int8); record any other operator failures. - -### Task 2: Fix Join's async merge - -**Files:** Modify `src/orcapod/core/operators/join.py` - -- [ ] Add helpers: - -```python -@staticmethod -def _merge_tag_tables(left: TagProtocol, right: TagProtocol) -> "pa.Table | None": - """Left tag columns then right-only ones; a shared key takes the right field.""" - left_table, right_table = left.as_table(), right.as_table() - fields, arrays = [], [] - for name in left_table.column_names: - if name in right_table.column_names: - lf, rf = left_table.schema.field(name), right_table.schema.field(name) - if arrow_utils.normalize_to_large_types(lf.type) != arrow_utils.normalize_to_large_types(rf.type): - raise InputValidationError(...) - fields.append(rf); arrays.append(right_table.column(name)) - else: - fields.append(left_table.schema.field(name)); arrays.append(left_table.column(name)) - for name in right_table.column_names: - if name not in left_table.column_names: - fields.append(right_table.schema.field(name)); arrays.append(right_table.column(name)) - if not fields: - return None # tagless inputs: Tag({}).as_table() has no columns and no rows - return pa.Table.from_arrays(arrays, schema=pa.schema(fields)) -``` - - `_merge_data(left_data, right_data)` hstacks `left.as_table()` and `right.as_table()` - (raising `InputValidationError` on overlapping names), merges `source_info()` dicts, and - returns `Data(table, source_info=..., data_context=left_data.data_context)`. - `_build_tag(table, system_tags, data_context)` returns `Tag(table, system_tags=..., - data_context=...)`, or `Tag({}, ...)` when `table is None`. -- [ ] Rewrite `_merge_pair_rename`, `_merge_pair_passthrough` and `_rename_sys_tags` on these - helpers (context: `left_tag.data_context` / `tag.data_context`). -- [ ] Run Task 1 tests: Join cases PASS. -- [ ] Commit `fix(join): async path builds merged rows from Arrow tables (ITL-837)`. - -### Task 3: Other operators - -- [ ] For each non-Join failure: same defect + small/local → fix here; otherwise mark - `pytest.mark.xfail(strict=True, reason=...)` and record for worker_done. - -### Task 4: Hash stability proof - -- [ ] Pinned test: for the issue's example (k: int16, a/b: int8), async and sync per-row - Tag/Data hashes equal the literal sync values recorded on main before the fix; and the sync - stream's `content_hash()` / `pipeline_hash()` equal literals recorded on main. -- [ ] Run `uv run pytest tests/test_hashing tests/ -k "stab or hash"` and the full suite. - -### Task 5: Docs and checks - -- [ ] CHANGELOG v0.2.0 entry; DESIGN_ISSUES entry (resolved, with **Fix:**); operators.md if - it describes async behaviour. -- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, - `uv run --with basedpyright basedpyright` (exit codes), `uv run pytest tests/` on 3.11, - 3.12, 3.13. -- [ ] Rebase on origin/main, push, PR "Fixes ITL-837", CodeRabbit rounds, worker_done. diff --git a/superpowers/plans/2026-11-10-itl-845-per-row-source-info.md b/superpowers/plans/2026-11-10-itl-845-per-row-source-info.md deleted file mode 100644 index fb857576a..000000000 --- a/superpowers/plans/2026-11-10-itl-845-per-row-source-info.md +++ /dev/null @@ -1,190 +0,0 @@ -# ITL-845 Per-row source info when materializing rows Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Every row that an operator, operator node or the sync orchestrator materializes into a -stream keeps its own `_source_*` provenance tokens instead of row 0's. - -**Architecture:** Both materializers (`StaticOutputOperatorPod._materialize_to_stream` and -`SyncPipelineOrchestrator._materialize_as_stream`) already concatenate each row's -`data.as_table(columns={"source": True})`, so the per-row `_source_*` columns are in the table. -The bug is the extra `source_info=rows[0][1].source_info()` argument: `ArrowTableStream` → -`arrow_utils.prepare_prefixed_columns` lets a concrete mapping value override the table's column -and broadcasts it. The fix drops that argument so the columns pass through, and moves the shared -row-to-table step into one helper used by both materializers. - -**Tech Stack:** Python 3.11–3.13, PyArrow, pytest, `uv run`. - ---- - -## Reproduction (before the fix, `ab544982`) - -Script: three rows with tokens `::row_0::x`, `row_1`, `row_2` (hash prefix elided). - -| Path | Output `_source_x` per row | -|---|---| -| async `PolarsFilter` | `row_0`, `row_0`, `row_0` | -| async `Batch(2)` (4 rows) | `[row_0, row_0]`, `[row_2, row_2]` | -| async `Batch(0)` | `[row_0 ×4]` | -| async `GroupBy(g)` | `[row_0, row_0]`, `[row_0, row_0]` | -| async `MergeJoin` | `[row_0, row_0]` ×3 | -| sync `Index(x, 0)` | `row_0::x[0]` ×3 | -| sync `Pick(scores, speed)` | `row_0::scores['speed']` ×3 | -| `OperatorJobNode.execute` (LOG) stored, `Select` | `row_0` ×3 | -| `OperatorJobNode.execute` (LOG) stored, `Batch(2)` | `[row_0,row_1]`, `[row_0,row_1]` (batch `['c']` gets 2 tokens) | -| `OperatorJobNode.async_execute` (LOG) stored | same broadcast | -| `SyncPipelineOrchestrator._materialize_as_stream` | `row_0` ×3 | - -## Nullability and hash invariants - -Before the fix, a broadcast concrete value made the `_source_*` field non-nullable; a `None` -row-0 value passed the column through (nullable). `Data.as_table` declares its source columns -nullable. To keep stored schemas and Arrow types identical, the helper sets each `_source_*` -field's nullability to "has nulls" — identical to the old flag whenever the old values were -correct, and the arrow hasher includes nullability in the digest. - -Prototype (monkeypatched, scratchpad only) vs. `ab544982`, operator node in LOG mode: - -| Case | stored schema | data columns | `_record_hash` unchanged | -|---|---|---|---| -| `execute` Select | same | same | 1 of 3 (row 0) | -| `execute` Batch(2) | same | same | 1 of 2 | -| `async_execute` Select | same | same | 1 of 3 | -| `async_execute` Batch(2) | same | same | 0 of 2 (row 0 was wrong too) | - -Unchanged by design: `Data.content_hash` (data columns only), function-node `record_id` -(system tags + input data hash), every `pipeline_hash`, every data value and Arrow type. -Changed because the values were wrong: `OperatorJobNode` `_record_hash` (hashes the stored row -including `_source_*`) for every row whose tokens were wrong, and the `content_hash` of the -producer-less intermediate stream `_materialize_to_stream` returns (`as_table(all_info=True)`). -**Decision gate:** the `_record_hash` change is raised to the coordinator before implementing. - -## File structure - -- Modify: `src/orcapod/core/operators/static_output_pod.py` — new static helper - `_rows_to_table(rows) -> tuple[pa.Table, tuple[str, ...]]`; `_materialize_to_stream` uses it - and no longer passes `source_info`. -- Modify: `src/orcapod/pipeline/sync_orchestrator.py` — `_materialize_as_stream` non-empty path - uses `StaticOutputOperatorPod._rows_to_table`, keeps `producer`/`upstreams`. -- Callers unchanged (`base.py`, `batch.py`, `index.py`, `pick.py`, `operator_node.py`): they all - go through `_materialize_to_stream`. -- Create: `tests/test_core/operators/test_materialize_source_info.py`. -- Docs: `DESIGN_ISSUES.md` O3 resolved; `CHANGELOG.md` one `### Fixed` entry. - -### Task 1: Failing regression tests - -**Files:** Create `tests/test_core/operators/test_materialize_source_info.py` - -- [ ] **Step 1:** Write tests. Helpers: `_source(n)` builds an `ArrowTableSource` with tags - `id` (large_string), `g` (int64, `i % 2`) and data `x` (int64), all non-nullable; - `_tokens(rows, col)` returns `[d.source_info()[col] for _, d in rows]`; `_run_async(op, *streams)` - feeds each stream into a `Channel` and collects `op.async_execute` output. Expected tokens come - from the input stream itself (`list(src.iter_data())`), so the test never hard-codes hashes. - Tests: - - `_materialize_to_stream` round-trip: scalar tokens, list tokens (rows from sync `Batch(2)`), - single row, rows whose source info is `None` (`Data` built with `source_info={"x": None}` and - mixed `None`/token rows), and that the `_source_*` Arrow fields keep the old nullability - (non-null when no row is `None`). - - async `PolarsFilter`, `SelectDataColumns` (UnaryOperator barrier): row `i` keeps token `i`. - - async `Batch(2)`, `Batch(0)`: each batch's list equals its members' tokens in order. - - async `GroupBy(by=["g"])`: each group's list equals the tokens of its members. - - async `MergeJoin` of two sources with distinct tokens: equals the sync `MergeJoin` output. - - general parity: for each operator above, async tokens == sync `op(stream)` tokens. - - sync `Index` on a non-null `large_list` column: `"{token_i}[0]"` per row. - - sync `Pick` on a `dict[str, int]` `DictSource`: `"{token_i}['speed']"` per row. - - `OperatorJobNode` LOG mode, `execute` and `async_execute`, `Select` and `Batch(2)`: stored - `_source_x` per row equals the input tokens (lists for Batch, one token per member). - - `SyncPipelineOrchestrator._materialize_as_stream(buf, source_node)` keeps per-row tokens and - the stream's `pipeline_hash`/`content_hash` still equal the node's. - - Zero-row: `_materialize_to_stream([])` still raises `ValueError`; async `Batch` and - `PolarsFilter` on an empty channel emit nothing; `_materialize_as_stream([], node)` builds - an empty stream. - - Pinned hash: for a fixed source, the `OperatorJobNode` `execute` stored row 0 - `_record_hash`, the data columns, every `pipeline_hash` and the output stream schema equal - the values recorded on `ab544982`. -- [ ] **Step 2:** Run `uv run pytest tests/test_core/operators/test_materialize_source_info.py -v`; - expect the per-row token tests to FAIL with row-0 tokens, zero-row/pinned tests to pass. -- [ ] **Step 3:** Commit `test(operators): pin per-row source info through materialization (ITL-845)`. - -### Task 2: Fix `_materialize_to_stream` - -**Files:** Modify `src/orcapod/core/operators/static_output_pod.py:194-232` - -- [ ] **Step 1:** Replace the body: - -```python - @staticmethod - def _rows_to_table( - rows: Sequence[tuple[TagProtocol, DataProtocol]], - ) -> "tuple[pa.Table, tuple[str, ...]]": - """Concatenate (Tag, Data) pairs into one table, keeping per-row source info. - - Each row contributes its own system-tag and ``_source_*`` columns, so every - row keeps its own provenance tokens. A ``_source_*`` field is declared - non-nullable unless some row's token is ``None``. - - Args: - rows: Non-empty sequence of (Tag, Data) pairs. - - Returns: - The concatenated table and the user tag column names. - - Raises: - ValueError: If ``rows`` is empty. - """ - if not rows: - raise ValueError("Cannot materialize an empty list of rows into a stream") - tag_tables = [tag.as_table(columns={"system_tags": True}) for tag, _ in rows] - data_tables = [data.as_table(columns={"source": True}) for _, data in rows] - # Rows may list the same columns in a different order (cached vs - # freshly computed), so match columns by name. - combined_tags = arrow_utils.concat_tables_by_name(tag_tables) - combined_data = arrow_utils.concat_tables_by_name(data_tables) - fields = [ - field.with_nullable(combined_data.column(i).null_count > 0) - if field.name.startswith(constants.SOURCE_PREFIX) - else field - for i, field in enumerate(combined_data.schema) - ] - combined_data = combined_data.cast( - pa.schema(fields, metadata=combined_data.schema.metadata) - ) - full_table = arrow_utils.hstack_tables(combined_tags, combined_data) - return full_table, tuple(rows[0][0].keys()) - - @staticmethod - def _materialize_to_stream(rows): - table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) - return ArrowTableStream(table, tag_columns=tag_keys) -``` - -- [ ] **Step 2:** Run the new test file; scalar/list/node tests pass, orchestrator test still fails. - -### Task 3: Fix the sync orchestrator - -**Files:** Modify `src/orcapod/pipeline/sync_orchestrator.py:170-235` - -- [ ] **Step 1:** Non-empty path: `full_table, user_tag_keys = StaticOutputOperatorPod._rows_to_table(buf)`; - construct `ArrowTableStream(full_table, tag_columns=user_tag_keys, producer=..., upstreams=...)` - without `source_info`. Update the docstring. -- [ ] **Step 2:** Run the new file — all pass. Commit `fix(operators): keep each row's source info when materializing rows into a stream (ITL-845)`. - -### Task 4: Hash and stability evidence - -- [ ] Run `uv run pytest tests/test_hashing tests/test_pipeline/test_system_tag_run_stability.py tests/test_channels tests/test_pipeline -q`. -- [ ] Rerun the scratchpad before/after comparison against the real fix (schema, data, record IDs). - -### Task 5: Docs - -- [ ] `DESIGN_ISSUES.md` O3: status `resolved`, `**Fix:**` note. -- [ ] `CHANGELOG.md`: one `### Fixed` entry under `[Unreleased]`, stating stored tables written by - earlier versions keep row 0's tokens for later rows and are not repaired, and operator-node - `_record_hash` values differ for those rows. -- [ ] Commit `docs: record ITL-845 fix`. - -### Task 6: Verification and PR - -- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit codes). -- [ ] `uv run --python 3.11|3.12|3.13 pytest tests/`. -- [ ] After ITL-837 merges: merge `origin/main`, remove the strict xfails this fix makes pass, rerun. -- [ ] Push, open PR (`Fixes ITL-845`), CodeRabbit rounds, CI green. diff --git a/superpowers/plans/2026-11-10-itl-847-semijoin-nested-columns.md b/superpowers/plans/2026-11-10-itl-847-semijoin-nested-columns.md deleted file mode 100644 index 012056251..000000000 --- a/superpowers/plans/2026-11-10-itl-847-semijoin-nested-columns.md +++ /dev/null @@ -1,115 +0,0 @@ -# SemiJoin over nested columns and string/large_string keys — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `SemiJoin` works when either input has list, large_list, struct or logical -(extension) columns, as non-key or key, and when a key is `string` on one side and -`large_string` on the other (ITL-847, first case of ITL-829), with sync and async paths -yielding the same rows. - -**Architecture:** The sync path replaces pyarrow's Acero `left semi` join (which rejects -nested non-key fields and `string` vs `large_string` keys) with a Polars -`join(how="semi", maintain_order="left")` over the left table and the right table's key -columns, then casts the result back to the left table's exact Arrow schema (types, -top-level and nested nullability, extension types). Before the join every common key's -two Arrow types go through `require_shared_arrow_type` (large-normalized equality, else -`InputValidationError`), because Polars would silently unify `int8` with `int16`. The -async path does the same type check on the first row of each side and builds its -lookup from hashable Arrow storage values instead of `as_dict()` values (struct and list -keys are unhashable dicts and lists today). - -**Tech Stack:** pyarrow 23, polars 1.41, pytest. - -## Verified prototype facts (polars 1.41.2, pyarrow 23.0.1) - -- Polars semi join keeps struct, list, large_list and orcapod extension (`Path`) payloads; - `.cast(left_schema)` restores the left table exactly (`equals` is True), including - `fixed_size_binary[16]` record ids and non-nullable fields. -- `string` vs `large_string` keys match. -- `int8` vs `int16` keys are silently unified by Polars: the explicit type check is - required. -- Scalar null keys never match (same as Acero today). A top-level null **struct/list** key - matches a null on the other side in Polars. To keep one rule ("a null key never - matches"), drop right rows whose key has a top-level null before the join; the async - path skips those rows too. Nulls nested inside a struct/list key compare equal in both - paths. Polars treats NaN as equal to NaN; the async freeze maps NaN to a sentinel. - -## Output types (decision) - -The result keeps the left input's Arrow schema exactly — today's behaviour, Join's -behaviour (`restore_schema_nullability` restores the reference type), and what the async -path yields, which passes left datagrams through untouched. Inputs that already worked -therefore produce byte-identical tables. (Re-typing to large variants on the sync path -alone would make sync and async disagree.) - -## Out of scope (report only) - -- ITL-829 second case (non-nullable vs nullable key, raised by `schema_utils` - `intersection_schemas` in `validate_binary_inputs`): unchanged; its strict xfail stays. -- Meta columns in SemiJoin (ITL-876). Join, MergeJoin. - -## Files - -- Modify: `src/orcapod/core/operators/semijoin.py` (sync + async). -- Modify: `tests/test_core/operators/test_async_sync_equivalence.py` (un-xfail - `test_semijoin_with_nested_columns`; the semijoin case may use nested sources). -- Create: `tests/test_core/operators/test_semijoin_nested.py` (public-interface tests). -- Modify: `DESIGN_ISSUES.md` (one appended entry). - -### Task 0: Before-dump - -- [ ] Write `scratchpad/dump_semijoin.py` that runs every SemiJoin case that works on main - (the `test_operators.py` fixtures, the equivalence-test `source_c`/`source_b(nested=False)` - pair, scalar-only pairs with string keys of the same type, empty-left/empty-right, - no-common-key) and dumps `content_hash`, `pipeline_hash`, `output_schema(all_info=True)`, - `as_table(all_info=True)` schema + rows, and each row's Tag/Data `content_hash`, sync and - async, to JSON. Run on main → `before.json`. - -### Task 1: Failing tests - -- [ ] Write `tests/test_core/operators/test_semijoin_nested.py`: - - kept-side nested non-key: list, large_list, struct, dataclass, Path columns on left; - result rows = filtered left rows in left order; `as_table()` schema equals the left's. - - filtering-side nested non-key: same columns on the right. - - struct and list keys (incl. nested null inside key, top-level null never matches). - - Path key. - - zero-row left, zero-row right (with nested columns). - - `CSVSource` x `DictSource` string keys (ITL-829 case 1), both orders. - - `int8` vs `int16` key raises `InputValidationError` naming the column and both types, - sync and async. - - predicted `output_schema` equals `binary_static_process(...).output_schema` (schema - prediction exception) for nested inputs. - - async equals sync for each nested scenario (reuse `assert_equivalent` helpers). - - row order = left order with a right side in a different order. -- [ ] Run: `uv run pytest tests/test_core/operators/test_semijoin_nested.py -v` → FAIL - (`ArrowInvalid ... not supported in join non-key field`, `Incompatible data types`). - -### Task 2: Sync fix - -- [ ] In `binary_static_process`: get `left_table` (source + system tags) and - `right_table`; for each common key call `require_shared_arrow_type("SemiJoin", key, ...)`; - `right_keys = right_table.select(common_keys)` with rows dropped where any key is null - (`pc.is_valid` and-ed); `pl.DataFrame(left_table).join(pl.DataFrame(right_keys), - on=common_keys, how="semi", maintain_order="left").to_arrow().cast(left_table.schema)`. - -### Task 3: Async fix - -- [ ] On the first left row, with right rows non-empty: compute common keys as today, call - `require_shared_arrow_type` on the first left and first right row's Arrow schemas, build - the lookup from `_row_key(tag, data, keys)` which reads each key from the datagram's - Arrow table (extension → `.storage`), `to_pylist`, freezes lists/dicts/tuples into - tuples and NaN into a sentinel, and returns `None` when any key is null (never matches). - -### Task 4: Un-xfail and verify - -- [ ] Remove the strict xfail on `test_semijoin_with_nested_columns`; switch the semijoin - equivalence case to nested sources; remove `semijoin` from `SCALAR_META_ONLY` only if the - meta test then passes (else leave, ITL-876). -- [ ] After-dump; `diff before.json after.json` must be empty. -- [ ] `uv run pytest tests/`, ruff check/format, basedpyright exit code. -- [ ] Confirm `test_binary_op_delegation` (ITL-829 case 2) still xfails strictly. - -### Task 5: Finish - -- [ ] DESIGN_ISSUES.md entry; commit; merge origin/main; push; PR (Fixes ITL-847, Refs - ITL-829 case 1); link; worker_done. diff --git a/superpowers/plans/2026-11-10-itl-849-chained-aggregation-source-info.md b/superpowers/plans/2026-11-10-itl-849-chained-aggregation-source-info.md deleted file mode 100644 index 59fc82d6b..000000000 --- a/superpowers/plans/2026-11-10-itl-849-chained-aggregation-source-info.md +++ /dev/null @@ -1,206 +0,0 @@ -# ITL-849 Chained aggregation (nested source info) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `GroupBy` and `Batch` applied to a stream whose data and source-info columns are -already list-valued (after `MergeJoin`, `Batch` or `GroupBy`) produce a well-typed -list-of-lists result, on the sync and async paths, with no hash change for any pipeline that -worked before. - -**Architecture:** The aggregation itself is already correct: `build_aggregated_table` types -`_source_x` as `large_list>`. The crash is downstream of it. -`ArrowTableStream.iter_data` converts each row's source info to Python values and passes them -to `Data`. `Data` then rebuilds them in `arrow_utils.prepare_prefixed_columns`, whose -value-supplied branch hard-types every list token as `pa.list_(pa.large_string())`, so a -second level of nesting raises `ArrowTypeError: Expected bytes, got a 'list' object`. The fix -has two parts: - -1. `iter_data` hands `Data` each row's source info as Arrow columns. These take the - pass-through branch, so the types are kept and nothing is inferred. -2. The value branch types Python-supplied tokens with the documented source-info rule: - `str`/`None` become `large_string` and lists become `large_list` of their element, - recursively. Today that rule lives in `tag_data._source_info_arrow_type`; it moves to - `arrow_utils.source_info_arrow_type` as the single definition. The async barrier path - (`_materialize_to_stream`, which passes `rows[0].source_info()` as Python values) reaches - this branch. - -**Tech Stack:** Python, PyArrow, pytest (`uv run`). - -**Approvals:** The coordinator approved this design on 2026-11-10. For the async per-row -source-info assertion it chose option A, a strict xfail citing ITL-845: `_materialize_to_stream` -broadcasts row 0's source info, which is ITL-845's bug in `static_output_pod.py`, a file this -PR must not edit. Whichever of the two PRs merges second removes the xfail. - ---- - -## Root cause evidence (Phase 1, on `main` at ab544982) - -- Reproduction: `GroupBy(["id"])(MergeJoin()(A, C)).as_table(all_info=True)` fails with - `arrow_table_stream.py:223 as_table` → `:299 iter_data` → `tag_data.py:338 Data.__init__` - → `arrow_utils.py:873 prepare_prefixed_columns` → `ArrowTypeError`. -- All five allowed chains fail at that frame, sync and async: MergeJoin→GroupBy, - MergeJoin→Batch, Batch→Batch, GroupBy→Batch and GroupBy→GroupBy. - `GroupBy(["g"])(Batch(...)(A))` is rejected with `InputValidationError`, as intended. -- Hypothesis test: patching only the value branch's list type in memory made every chain - work. Types were exact (`large_list>`; `float32` kept; - `Path` → `list[list[orcapod.path]]`), the prediction equalled `static_process` output, the - zero-row chains worked, and hashes were computable. -- Hash baseline for 14 pipelines that already worked: - `scratchpad/hash_snapshot.py` → `hash_before.json`. It is deterministic across two runs. - -## File map - -- Modify `src/orcapod/utils/arrow_utils.py`: add `source_info_arrow_type`, and make the - `prepare_prefixed_columns` value branch use it. -- Modify `src/orcapod/core/datagrams/tag_data.py`: remove `_source_info_arrow_type` and use - `arrow_utils.source_info_arrow_type`. `_source_info_python_type` stays and its docstring is - updated. -- Modify `src/orcapod/core/streams/arrow_table_stream.py` (`iter_data`): batch the data - columns together with the source-info columns and pass Arrow slices to `Data`. -- Create `tests/test_core/operators/test_chained_aggregation.py`: chain regressions (sync, - async, zero-row, validation). -- Modify `tests/test_utils/test_arrow_utils.py`: value-branch unit test. -- Modify `tests/test_core/streams/test_iter_data_chunk_layouts.py`: nested source info - across chunk layouts. -- Docs: `DESIGN_ISSUES.md` (one entry at the end), `CHANGELOG.md` (one entry in - `[Unreleased]`, which is the v0.2.0 section), and the nested-list rule in `CLAUDE.md` and - `.zed/rules`. - ---- - -### Task 1: Failing chain regressions - -**Files:** Create `tests/test_core/operators/test_chained_aggregation.py`. - -- [ ] **Step 1: Write the tests.** Sources: - - `A`: tags `id: int64`, `g: int64`; data `x: int8`, `y: float32`, `name: string`, - `lname: large_string`, `lst: list`, `p: Path` (logical); all non-nullable. - - `C`: tags `id`, `g`; data `x: int8`, `z: large_string`, `p: Path`. - - `C0`: `C` with disjoint ids, used for zero-row inputs. - - Chains are `(name, upstream factory, operator factory)`: - - `mergejoin_group_by`: `MergeJoin()(A, C)` → `GroupBy(["g"])` - - `mergejoin_batch`: `MergeJoin()(A, C)` → `Batch(2)` - - `batch_batch`: `Batch(2)(A)` → `Batch(2)` - - `group_by_batch`: `GroupBy(["g"])(A)` → `Batch(2)` - - `group_by_group_by`: `GroupBy(["g"])(A)` → `GroupBy(["g"])` - - Assertions, each case parametrized over the chains: - - The issue's reproduction: - `GroupBy(["id"])(MergeJoin()(A, C)).as_table(all_info=True)` succeeds. - - Every data column has an exact expected Arrow type, written as a literal per chain - (for example, MergeJoin→GroupBy: `x` is `large_list>`, `y` is - `large_list`, `name` is `large_list`, `lst` is - `large_list>`, `p` is the converter's `list[list[Path]]` type). Every - member column is non-nullable. - - Every `_source_` column is `large_list` of its incoming source type: one level - deeper than upstream, and non-nullable. - - `op.output_schema(up, columns=c) == op.unary_static_process(up).output_schema(columns=c)` - for `c` in `({}, {"system_tags": True})`. - - The values are exact for `x`, `y` and `lst`, and `p` round-trips to `Path` objects. - - Per row, `data.source_info()[col]` equals that row's `_source_` value in - `as_table(columns={"source": True})`. It is a list for every row, and each member's tokens - name that member's upstream rows. - - `data.content_hash()`, `tag.content_hash()`, the record-ID preimage hash, the stream - `content_hash()` and the stream `pipeline_hash()` all compute. - - Async: feed the upstream rows into a `Channel`, run `op.async_execute`, then compare each - row with the sync result. The data and tag content hashes must be equal, the - `as_table(columns={"source": True})` Arrow types equal, and the values equal. - - Async per-row source info equals sync. This assertion is marked - `xfail(strict=True, reason="ITL-845: _materialize_to_stream broadcasts rows[0] source info")`. - - Async three-stage chain: upstream `Batch(2)(GroupBy(["g"])(A))`, then `Batch(1)` async. - This reaches the value branch with nested Python tokens; the types must match sync. - - Zero-row: `MergeJoin()(A, C0)` → GroupBy, → Batch, and → Batch → Batch. There are zero - rows, the types equal those of the non-empty chain, and the prediction matches. - - `GroupBy(["g"])(Batch(2)(A))` raises `InputValidationError`. -- [ ] **Step 2: Run them and confirm they fail with `ArrowTypeError`.** - `uv run pytest tests/test_core/operators/test_chained_aggregation.py -q` - -### Task 2: Single source-info Arrow type rule, and value branch fix - -**Files:** `src/orcapod/utils/arrow_utils.py`, `src/orcapod/core/datagrams/tag_data.py`, -`tests/test_utils/test_arrow_utils.py`. - -- [ ] **Step 1: Write the failing unit test.** - `prepare_prefixed_columns(table, {"_source_": {"v": [["a"], ["b", "c"]]}})` gives - `_source_v: large_list>` with that value in every row. - `ArrowTableStream(..., source_info={"v": [["a"], ["b"]]})` iterates. A flat list still - gives `large_list`, a string gives `large_string`, and `None` gives a nullable - `large_string`. -- [ ] **Step 2: Run it and confirm it fails.** -- [ ] **Step 3: Implement.** In `arrow_utils`: - - ```python - def source_info_arrow_type(value: Any) -> pa.DataType: - """Return the Arrow type of one source-info provenance token. ...""" - if isinstance(value, list): - if not value: - return pa.large_list(pa.large_string()) - return pa.large_list(source_info_arrow_type(value[0])) - return pa.large_string() - ``` - - Keep the TODO(NPIPE-204) comment on it. In the value branch, replace the - `Collection`/`pa.list_` special case with - `column_values = pa.array([value] * num_rows, type=source_info_arrow_type(value))`. - In `tag_data.py`, delete `_source_info_arrow_type` and call - `arrow_utils.source_info_arrow_type`. -- [ ] **Step 4: Run the unit test and confirm it passes.** - -### Task 3: `iter_data` passes source info as Arrow columns - -**Files:** `src/orcapod/core/streams/arrow_table_stream.py`, -`tests/test_core/streams/test_iter_data_chunk_layouts.py`. - -- [ ] **Step 1: Write the failing test.** A stream whose `_source_x` column is - `large_list>`, chunked differently from the tag and data columns, - yields the right nested source info for every row. -- [ ] **Step 2: Implement.** Build the batching table once, with the source-info columns - appended to `self._table`, so `to_batches()` still cuts at the union of all chunk - boundaries and every row's source info is indexed by its absolute row: - - ```python - source_columns = list(self._source_info_table.column_names) - batching_table = self._table - for name in source_columns: - batching_table = batching_table.append_column( - self._source_info_table.schema.field(name), self._source_info_table.column(name) - ) - for batch in batching_table.to_batches(): - tag_batch = batch.select(tag_columns) - data_batch = batch.select(data_columns + source_columns) - ... - data = Data(data_batch.slice(i, 1), data_context=self.data_context) - ``` - - This removes `source_info_rows` and `row_offset`. -- [ ] **Step 3: Run the chunk-layout and chain tests and confirm they pass,** apart from the - strict xfail. - -### Task 4: Hash evidence and full verification - -- [ ] Run `hash_snapshot.py` into `hash_after.json` and `diff` it against `hash_before.json`. - The diff must be empty. -- [ ] `uv run pytest tests/test_hashing tests/test_pipeline/test_system_tag_run_stability.py tests/test_core/streams tests/test_core/operators -q` -- [ ] `uv run pytest tests/` on Python 3.11, 3.12 and 3.13 (`uv run --python 3.1x`). -- [ ] Run ruff check, ruff format --check and basedpyright, and record each exit code. -- [ ] Commit: `fix(streams): chained aggregation over list-valued source info (ITL-849)`. - -### Task 5: Docs - -- [ ] `DESIGN_ISSUES.md`: one self-contained entry at the end, status `resolved`, with a - **Fix:** note. -- [ ] `CHANGELOG.md`: one `####` entry in `[Unreleased]`, before `### Breaking Changes`. -- [ ] `CLAUDE.md` and `.zed/rules`: an aggregated source-info column is `large_list` of its - incoming source-info type (`large_list` for scalar tokens, one level deeper - for each further aggregation). Data source-info values may be nested lists. - `prepare_prefixed_columns` types Python tokens only through `source_info_arrow_type`. -- [ ] Commit: `docs: record ITL-849 nested source-info rule`. - -### Task 6: PR - -- [ ] Fetch main, merge or rebase if it moved, and rerun the suite. -- [ ] Push, then open the PR with sensei:create-pr. The body is "Fixes ITL-849" and ends with - the Claude Code line. -- [ ] Run `orca linear attach` and `orca worktree set --pr`. -- [ ] Run the CodeRabbit rounds (at most 3) and get CI green. Then send worker_done. diff --git a/superpowers/plans/2026-11-10-itl-854-join-async-meta-columns.md b/superpowers/plans/2026-11-10-itl-854-join-async-meta-columns.md deleted file mode 100644 index 3e10fa773..000000000 --- a/superpowers/plans/2026-11-10-itl-854-join-async-meta-columns.md +++ /dev/null @@ -1,58 +0,0 @@ -# ITL-854: async Join keeps Data meta columns — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** `Join.async_execute` emits the same `Data` meta (`__`-prefixed) columns as `Join.static_process`: names, collision renames, Arrow types, nullability, values and order. The fix goes into PR #280 (ITL-837). - -**Architecture:** Extract the collision-rename loop of `static_process` into one static helper, `Join._collision_renames`. `static_process` calls it unchanged, so the sync path keeps its exact behaviour. The async `_merge_data` reads each side with `as_table(columns={"meta": True})`, renames the right side's colliding columns with the same helper, using the right input's canonical index (1 for two inputs, i for the i-th binary join of the staggered chain), and hstacks the Arrow tables. `Data` splits the meta columns back out. No Python-value round-trip, no new operator state, no protocol change. - -**Tech Stack:** pyarrow, pytest, uv. - -**Decided with the coordinator (November 10, 2026, option A):** async meta order is canonical concatenation, which is the physical order sync produces. Sync moves columns that an input's `output_schema()` lists as data to the front, and only a raw `ArrowTableStream` over a table with `__` columns lists them. So async differs only for such inputs when their meta collides or is mixed with other stream kinds. `async_execute` never sees input schemas, so that order case is pinned with a strict xfail naming ITL-858. The pre-existing sync rename bug is pinned with its own strict xfail (ITL-857) and left unfixed: the rename does not check the next input's own unrenamed names, so `__x` renamed to `__x_1` can duplicate that input's own `__x_1`. - ---- - -## Task 1: Failing tests (meta-column equivalence) - -**Files:** Modify `tests/test_core/operators/test_async_sync_equivalence.py` - -- [ ] Add `assert_same_meta(expected, actual)`. It compares `data.arrow_schema(columns={"meta": True})` (types, nullability, order), `data.meta_columns`, and the meta values from `data.as_table(columns={"meta": True}).select(meta_columns).to_pylist()`. -- [ ] Add `_with_meta(source, meta: dict[str, Column]) -> ArrowTableStream`. It builds a stream from `source.as_table(columns={"source": True, "system_tags": True})` with the meta columns appended, so every row's `Data` carries them. -- [ ] Add `META_CASES`: every `Case` with its sources wrapped by `_with_meta`, using distinct meta names per source. Types: non-null `int8`, nullable `string`, `list`. Add `test_async_meta_equals_sync` over `META_CASES`, through `process` (validation passes because nothing collides). -- [ ] Add `test_join_colliding_meta_async_equals_sync`, which runs sync through `static_process` (the only route that accepts colliding meta). Two inputs: `[__u_a, __m]` and `[__u_b, __m]`. Three inputs: `[__m]` each. -- [ ] Add a strict xfail `test_join_meta_order_schema_dependent` (ITL-858): three inputs `[__u_i, __m]` through `static_process`. Sync moves the unique names forward; async keeps concatenation order. -- [ ] Add a strict xfail `test_join_meta_rename_collides_with_own_column` (ITL-857): inputs whose canonical order makes `__x` rename to `__x_1` while that input already has `__x_1`. Sync raises Polars `DuplicateError`. The test asserts that both paths succeed and are equal. -- [ ] Run `uv run pytest tests/test_core/operators/test_async_sync_equivalence.py -q`. Expected: the Join meta cases FAIL (async meta `[]`). Record the results for the other operators. - -## Task 2: Shared helper and async fix - -**Files:** Modify `src/orcapod/core/operators/join.py` - -- [ ] Add a static method `_collision_renames(existing, incoming, keep, index) -> dict[str, str]` holding exactly the loop now at `static_process` lines 207-216: a candidate `f"{col}_{index}"`, with the counter bumped while the candidate is in `existing` or already used. -- [ ] `static_process`: `rename_map = self._collision_renames(set(table.column_names), next_table.column_names, join_key_set, idx)`. No other sync change. -- [ ] `_merge_data(left, right, right_index)`: keep the data-overlap check on `as_table()`. Then: - ```python - left_full = left.as_table(columns={"meta": True}) - right_full = right.as_table(columns={"meta": True}) - renames = Join._collision_renames(set(left_full.column_names), right_full.column_names, (), right_index) - if renames: - right_full = right_full.rename_columns([renames.get(n, n) for n in right_full.column_names]) - return Data(arrow_utils.hstack_tables(left_full, right_full), source_info=..., data_context=left.data_context_key) - ``` -- [ ] `_merge_pair_rename` passes `right_index=1`. `_merge_pair_passthrough` gains `right_index`, and `_staggered_join` binds `i` per binary join with `functools.partial`. -- [ ] Run the equivalence file: the Join meta cases PASS, and the xfails stay xfailed. - -## Task 3: Other operators - -- [ ] For each other operator with its own async path whose meta case fails: if the fix is the same small local change, make it. Otherwise pin it with a strict xfail naming ITL-846 and report it. - -## Task 4: Hash invariance and full verification - -- [ ] `uv run pytest tests/` on 3.11, 3.12 and 3.13 (`uv run --python 3.1x`). The pinned sync hashes in `test_issue_example_types_and_pinned_hashes` and every existing hash test must stay unchanged. -- [ ] `uv run --with ruff ruff check .`, `uv run --with ruff ruff format --check .`, `uv run --with basedpyright basedpyright` (exit codes). - -## Task 5: Docs and PR - -- [ ] CHANGELOG ITL-837 entry: add one sentence on meta columns (ITL-854). DESIGN_ISSUES ITL-837 entry: add one sentence. -- [ ] Commit (Conventional Commits, Co-Authored-By trailer), make one push, and update the PR #280 body: add `Fixes ITL-854` next to `Fixes ITL-837`, and one plain sentence on the remaining order difference. -- [ ] Wait for green CI, request a CodeRabbit review (at most 2 rounds), then send worker_done. diff --git a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md deleted file mode 100644 index 546c5e252..000000000 --- a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md +++ /dev/null @@ -1,174 +0,0 @@ -# ITL-864 DeltaTableDatabase read-your-writes Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Every read method of `DeltaTableDatabase` returns exactly the rows a flush would leave, through every handle of an `at()` family, so `node.get_all_records()` returns every row right after the first run on a Delta store. - -**Architecture:** Two defects, one contract. (1) `at()` handles share the root's pending batches, and any handle's `flush()` writes every pending key, but each handle keeps its own `_delta_table_cache`, so a handle that loaded a table earlier keeps serving a stale `DeltaTable` snapshot after a sibling flushed. The fix shares `_delta_table_cache`, `_existing_ids_cache` and `_cache_dirty` across the family, exactly as pending state is already shared. (2) The read methods merge pending and stored rows inconsistently (`get_records_by_ids` ignores pending; `get_records_with_column_value` ignores stored whenever a pending batch exists; overlapping ids show twice). The fix: every read returns stored rows plus pending rows whose record id is not stored (Delta's flush is `when_not_matched_insert_all`, so the stored row wins), `None` when empty. - -**Tech Stack:** Python, pyarrow, deltalake, pytest, `uv run`. - -Approved by the coordinator (option A, Nov 10, 2026). No public API, hash, stored schema or dependency change. Error handling is untouched (PR #290 / ITL-884 rewrites it in the same file; merge main after it lands). - ---- - -## Reproduction (Step 1 of the brief) - -`PipelineJob`, `DictSource` of 2 or 3 ints, `int -> int` pod, `DeltaTableDatabase`: - -| store | orchestrator | n | `get_all_records` | `iter_data` | -|---|---|---|---|---| -| delta | sync | 2 / 3 | 1 / 1 | 2 / 3 | -| delta | async | 2 / 3 | 1 / 2 | 2 / 3 | -| memory | both | 2 / 3 | 2 / 3 | 2 / 3 | - -Trace: the result-store handle (`at("pipeline", "_result")`) flushes the pipeline table's rows; the node's pipeline handle (`at("pipeline")`) cached the table at version 0 and still reads version 0 while the table is at version 2. Its pending batch is empty; a fresh handle reads all rows. - -## Contract and per-method findings (database level) - -Contract: a read through a handle reflects all writes of its `at()` family, pending and flushed together, and equals what a flush would leave: no duplicates, no missing rows. - -| case | `get_all_records` | `get_record_by_id` | `get_records_by_ids` | `get_records_with_column_value` | -|---|---|---|---|---| -| A sibling handle flushed | stale | stale | stale | stale | -| B stored + pending, one handle | ok | ok | misses pending | misses stored when pending exists; empty table instead of `None` | -| C pending id already stored, `skip_duplicates=False` | duplicate | returns pending row, flush keeps stored | (dup) | (dup) | -| D same, `skip_duplicates=True` | ok | ok | ok | ok | -| `record_id_column=` | returns (no hang) | | | | - -Other backends: `InMemoryArrowDatabase` shares `_tables` across `at()` and `ConnectorArrowDatabase` re-reads committed rows on every call, so neither goes stale; both merge pending + committed in every method. Both flush with the NEW row winning on id overlap (Delta keeps the stored row) and show both rows before flush under `skip_duplicates=False`. Report only. - -Thread safety: the async orchestrator runs only the user function in `run_in_executor` (`core/executors/local.py`); every database read and write happens on the event-loop thread, as for the already-shared pending dicts, so sharing the caches adds no cross-thread access. - -## File structure - -- Modify: `src/orcapod/databases/delta_lake_databases.py` — `__init__` / `at()` share caches; new helpers `_pending_not_stored` and `_combine_stored_and_pending`; the four read methods use them. -- Create: `tests/test_databases/test_delta_read_your_writes.py` — database-level regression tests. -- Create: `tests/test_pipeline/test_delta_node_records_after_run.py` — `PipelineJob` regression, both orchestrators. -- Modify: `DESIGN_ISSUES.md` — one entry appended at the end. - ---- - -### Task 1: Failing database-level tests - -**Files:** Create `tests/test_databases/test_delta_read_your_writes.py` - -- [ ] **Step 1: Write the tests** - -Helpers: `_rows(db, ids, vals, groups)` builds a table with `__record_id` (`large_binary`), `v` (`int64`), `g` (`int64`); `_values(table)` returns sorted `v` or `None`. A `READS` parametrization maps each read method to a call and its expected values: - -```python -READS = { - "get_all_records": (lambda db: db.get_all_records(PATH), [1, 2]), - "get_record_by_id": (lambda db: db.get_record_by_id(PATH, b"b"), [2]), - "get_records_by_ids": (lambda db: db.get_records_by_ids(PATH, [b"a", b"b"]), [1, 2]), - "get_records_with_column_value": ( - lambda db: db.get_records_with_column_value(PATH, {"g": 0}), [1, 2]), -} -``` - -Tests: -1. `test_sibling_handle_sees_rows_flushed_by_another_handle[method][flusher]`: `root = DeltaTableDatabase(tmp_path)`, `h1, h2 = root.at("s"), root.at("s")`; `h1` writes `a=1` with `flush=True` and reads once (loads its snapshot); `h2` writes `b=2`; read through `h1` and through `root` (path `("s",) + PATH`) before flush; flush through `flusher` in (`h2`, `root`, `h1`); read through `h1`, `root` and a new `root.at("s")` after flush. All equal the expected values. -2. `test_pending_and_stored_rows_are_both_returned[method]`: one handle, stored `a=1 (g=0), b=2 (g=1)`, pending `c=3 (g=1)`: `get_records_by_ids([a, c]) == [1, 3]`, `get_records_with_column_value({"g": 1}) == [2, 3]`, `{"g": 0} == [1]`, `get_all_records == [1, 2, 3]`, `get_record_by_id(c) == [3]`. -3. `test_no_match_returns_none`: stored + pending, `get_records_with_column_value({"g": 9})` and `get_records_by_ids([b"zz"])` are `None`. -4. `test_pending_row_with_stored_id_reads_as_flush_keeps_it[skip_duplicates]`: stored `a=1`; pending add of `a=10, d=4`; before flush each method returns `a=1` and `d=4` once (`get_all_records == [1, 4]`, `get_record_by_id(a) == [1]`, `get_records_by_ids([a, d]) == [1, 4]`, `get_records_with_column_value({"g": 0}) == [1, 4]`); after `flush()` the same values. -5. `test_record_id_column_is_returned_for_merged_rows`: stored + pending, `get_all_records(PATH, record_id_column="rid")` has column `rid` with all three ids and no `__record_id`; same for `get_records_by_ids` and `get_records_with_column_value`. - -- [ ] **Step 2: Run, verify the expected failures** - -Run: `uv run pytest tests/test_databases/test_delta_read_your_writes.py -v` -Expected: test 1 fails for every method after flush; test 2 fails for `get_records_by_ids` / `get_records_with_column_value`; test 3 fails (empty table, not `None`); test 4 fails for `skip_duplicates=False`. - -### Task 2: Failing PipelineJob test - -**Files:** Create `tests/test_pipeline/test_delta_node_records_after_run.py` - -- [ ] **Step 1: Write the test** - -```python -@pytest.mark.parametrize("orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) -def test_node_returns_every_row_right_after_first_run(tmp_path, orchestrator): - source = op.sources.DictSource([{"id": i, "x": 10 * i} for i in range(3)], tag_columns=["id"]) - - @op.function_pod(output_keys=["result"]) - def double(x: int) -> int: - return x * 2 - - job = PipelineJob(store=DeltaTableDatabase(tmp_path)) - with job: - double.pod(source, label="double") - job.run(orchestrator=orchestrator()) - - records = job.nodes["double"].get_all_records() - assert records is not None - assert sorted(records.column("result").to_pylist()) == [0, 20, 40] -``` - -- [ ] **Step 2: Run, verify it fails** (`1 != 3` rows) for both orchestrators. - -- [ ] **Step 3: Commit the failing tests** - -```bash -git add tests/test_databases/test_delta_read_your_writes.py tests/test_pipeline/test_delta_node_records_after_run.py -git commit -m "test(databases): regression tests for Delta reads missing own writes (ITL-864)" -``` - -### Task 3: Share the snapshot caches across the `at()` family - -**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (`__init__`, `at()`) - -- [ ] **Step 1:** Add keyword-only private constructor parameters `_shared_delta_table_cache`, `_shared_existing_ids_cache`, `_shared_cache_dirty`, used when given (same pattern as `_shared_pending_batches`); `at()` passes the root instance's dicts. Comment why: any family member's `flush()` writes every pending key and refreshes the cache entry for it, so the snapshot cache must be family-wide or a sibling keeps reading the old version. -- [ ] **Step 2:** Run Task 1 test 1 and the Task 2 test: expected PASS for test 1 and Task 2. - -### Task 4: Merge stored and pending rows the way flush does - -**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (read methods) - -- [ ] **Step 1:** Add helpers: - -```python -def _pending_not_stored(self, record_path): - """Pending rows whose record id is not stored; flush inserts only these.""" - pending = self._pending_batches.get(self._get_record_key(record_path)) - if pending is None or pending.num_rows == 0: - return None - stored_ids = self._get_existing_ids(record_path) - if stored_ids: - id_type = pending.schema.field(self.RECORD_ID_COLUMN).type - pending = pending.filter(pc.invert(pc.is_in( - pending[self.RECORD_ID_COLUMN], value_set=pa.array(list(stored_ids), type=id_type)))) - return pending if pending.num_rows > 0 else None - -def _combine_stored_and_pending(self, stored, pending, record_id_column): - parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] - if not parts: - return None - table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) - return self._handle_record_id_column(table, record_id_column) -``` - -- [ ] **Step 2:** `get_all_records`: pending part uses `_pending_not_stored`. `get_record_by_id`: serve from pending only when the id is pending and not stored. `get_records_by_ids`: stored read (existing try/except unchanged) plus pending filtered by the ids. `get_records_with_column_value`: stored read (existing try/except unchanged) plus pending filtered by the expression; `None` when empty. -- [ ] **Step 3:** Run `uv run pytest tests/test_databases tests/test_pipeline -q`: all pass. -- [ ] **Step 4:** Commit `fix(databases): Delta reads return every row the handle family wrote (ITL-864)`. - -### Task 5: Design issue entry, full verification, PR - -- [ ] Append one `DESIGN_ISSUES.md` entry (resolved, with **Fix:**; note that writes from other processes are still not seen until a handle reloads the table — follow-up for the coordinator to file). -- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (golden hashes unchanged). -- [ ] Merge `origin/main`, rerun if it moved, push, PR "Fixes ITL-864", `orca linear attach`, `orca worktree set --pr`. - ---- - -## Review rounds and continuation notes (successor-readable) - -Done and pushed (verify, do not redo): -- Round 1 (e9c78b5c): a failed ID refresh keeps the cache dirty; `get_records_by_ids` refresh shares the stored-read handler (8f9adf8b). -- Round 2 (8f9adf8b): refresh when the record key has no ID-cache entry; stored + pending combined with `normalize_table_to_large_types` + `pa.concat_tables(promote_options="default")` (b8bcc817). Tests: `test_fresh_handle_drops_pending_rows_with_stored_ids`, evolved-schema tests. - -Round 3 (CodeRabbit review of b8bcc817, outside-diff, `get_records_with_column_value`), approved fix: -- A side (stored schema, read without loading data; pending column names) that lacks a queried column contributes no rows; after a flush its rows hold null there and `col == value` never matches null. A column on neither side keeps today's `ArrowInvalid` ("No match for FieldRef"). -- Tests: `test_query_on_column_one_side_lacks_matches_read_after_flush[column_only_pending|column_only_stored]`, `test_query_on_column_neither_side_has_raises[flushed]`. -- Also the round-1 outside-diff trivial: `_add_to_pending_batch` cast `list[str]` -> `list[bytes]`. -- Reply: signed top-level PR comment (no thread for outside-diff findings), and one for the round-1 trivial. - -Then: merge origin/main (b197544e); wait for PR #290 (ITL-884, StoreReadError) to merge; merge main again, take #290's error handling around `_pending_not_stored` / `_combine_stored_and_pending`; a failed ID refresh must raise `StoreReadError` on every read method; full suite; worker_done only after the post-#290 merge is green. diff --git a/superpowers/plans/2026-11-10-itl-875-mit-license.md b/superpowers/plans/2026-11-10-itl-875-mit-license.md deleted file mode 100644 index 9ba36124d..000000000 --- a/superpowers/plans/2026-11-10-itl-875-mit-license.md +++ /dev/null @@ -1,88 +0,0 @@ -# ITL-875 (license part): relicense Orcapod under MIT — Implementation Plan - -**Goal:** From v0.2.0 Orcapod is MIT licensed, to be published on public PyPI as `orcapod` -from a public repository (developer decision, November 10, 2026). This PR applies only the -license: `LICENSE`, `pyproject.toml`, the packaging test, and every statement of the license or -of "internal registry only" outside the release docs. It reverses the proprietary license of -PR #277 (ITL-827). PR body says `Refs ITL-875` (the rest of the issue follows). - -**Out of scope (owned elsewhere):** `release.yml`, `CHANGELOG.md`, `RELEASING.md`, and the -release-docs content of PR #269 (ITL-810). Sentences there that MIT / public PyPI invalidates -are listed in the worker report, not edited. - -## Context - -- `pyproject.toml` declares `license = { text = "LicenseRef-Metamorphic-Proprietary" }` with no - license classifier. ITL-827 avoided a classifier because `pip-licenses --from=mixed` prefers - trove classifiers over the `License` field, and `License :: Other/Proprietary License` fails - the allow list. -- The build backend is `hatchling` (originally `hatchling>=1.21.0`, now `>=1.27.0`; not locked by `uv.lock`; `uv build` - resolves the latest, which emits Metadata-Version 2.5). `pip-licenses` is locked at 5.5.5. -- `_license-check.yml` runs `uv run pip-licenses --allow-only="MIT;...;LicenseRef-Metamorphic-Proprietary" --partial-match` - over the synced dev environment, which includes `orcapod` itself. - -## Decision: the `license` declaration form - -Preferred: PEP 639, `license = "MIT"` (SPDX expression), no license classifier (PEP 639 -forbids combining an expression with license classifiers), `LICENSE` picked up by hatchling's -default `license-files`. Metadata then carries `License-Expression: MIT` and -`License-File: LICENSE`. The build floor is `hatchling>=1.27.0` (see the revision note -below); Task 2 also builds pinned to that floor (constraints file `hatchling==1.27.0`) to confirm it emits -Metadata-Version 2.4 with the declaration. - -**Revised after review (CodeRabbit on #283, approved by the coordinator):** hatchling 1.21.0 -writes `License-Expression` under `Metadata-Version: 2.1`, which does not define the field. -The build floor is raised to `hatchling>=1.27.0` (metadata 2.4), and the packaging test -asserts the floor, a built Metadata-Version of at least 2.4, and the exact MIT text. - -Fallback (only if pip-licenses 5.5.5 does not report `MIT` for a `License-Expression`-only -distribution): `license = { text = "MIT" }` plus `License :: OSI Approved :: MIT License`. - -Verified empirically in Task 2 before committing. - -## Tasks - -### Task 1: failing test first -- `tests/test_packaging.py`: replace `test_license_is_metamorphic_proprietary` with - `test_license_is_mit`: `project.license == "MIT"`, no `License ::` classifier, and `LICENSE` - is the standard MIT text with the single copyright line - `Copyright (c) 2025-2026 NauticaLab` (holder decided by the developer, November 10, 2026; spelled `NauticaLab` per review comment 4534040178). Add `test_built_metadata_declares_mit` over the - existing `built_dists` fixture: wheel METADATA has `License-Expression: MIT` and no - `License:` field or license classifier; sdist PKG-INFO the same. -- Run; show both fail. - -### Task 2: apply the license -- `LICENSE`: standard MIT text (SPDX `MIT` template), copyright line exactly once. -- `pyproject.toml`: `license = "MIT"`; build floor `hatchling>=1.27.0`. -- Verify: `uv build` twice, once with the latest hatchling and once at the floor (write - `hatchling==1.27.0` to a constraints file, then `uv build --build-constraint `); inspect wheel METADATA and sdist PKG-INFO, confirm LICENSE in both; - `uv sync --locked --dev` then the `_license-check.yml` command verbatim, plus - `pip-licenses --packages orcapod` showing `MIT`. -- Run the packaging tests; green. - -### Task 3: statements of the license / distribution route -- `README.md` "Releasing": drop the internal package index sentence; say it is published on - PyPI as `orcapod` under MIT; add a short `## License` section. -- `docs/index.md` "Installation": `pip install orcapod` / `uv add orcapod`, drop the - "install command pending" note. -- `CLAUDE.md` / `.zed/rules`: only if they state the license (grep showed none). -- `DESIGN_ISSUES.md` DEP2 (ITL-827): append a dated note that the license decision was - reversed to MIT on November 10, 2026 (ITL-875). History unchanged. -- `_license-check.yml` keeps its allow list unchanged (the brief does not ask to edit it); - whether to drop `LicenseRef-Metamorphic-Proprietary` from it is reported. - -### Task 4: verification and report -- Four CI commands: ruff check, ruff format --check, basedpyright (exit code), full pytest. -- Report-only investigations: pip-licenses for every runtime dependency (default install, and - extras), non-MIT-compatible ones; scan of the tree for internal-only material (internal URLs, - hostnames, tokens, customer/dataset names); PR #269 sentences needing change. -- Merge origin/main if moved, push, open PR (`Refs ITL-875`), link via orca. - -### Task 5: review fix, holder spelling (comment 4534040178) -- The developer asked for the holder to be spelled `NauticaLab` (capital L). -- Failing test first: `COPYRIGHT_LINE` in `tests/test_packaging.py` becomes - `Copyright (c) 2025-2026 NauticaLab`; run `test_license_is_mit`, show it fails. -- `LICENSE` line 3 becomes `Copyright (c) 2025-2026 NauticaLab`; test green. -- Lower-case `nauticalab` in URLs and repository slugs is an identifier and stays unchanged. -- Rebuild wheel and sdist; the shipped LICENSE carries the new spelling and METADATA still - says `License-Expression: MIT`. Four CI commands, one commit, reply on the thread. diff --git a/superpowers/plans/2026-11-10-itl-878-store-write-failure.md b/superpowers/plans/2026-11-10-itl-878-store-write-failure.md deleted file mode 100644 index bf329d202..000000000 --- a/superpowers/plans/2026-11-10-itl-878-store-write-failure.md +++ /dev/null @@ -1,306 +0,0 @@ -# ITL-878: a failed store write fails the run — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this -> plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** When any node's write to the store fails, `PipelineJob.run()` (and the orchestrators -under it) raises a `StoreWriteError` naming the node and table, on both orchestrators and -independent of `error_policy`, instead of returning normally with the rows silently missing. - -**Status:** Option A proposed to the coordinator on 2026-11-10; the coordinator is taking it to -the developer (new public exception + documented behaviour change). Tasks 1 (failing tests) -may proceed now; Tasks 2+ wait for approval. - -**Architecture:** Every node-owned store write site converts a backend exception into -`orcapod.errors.StoreWriteError` (chained `from` the backend error). The per-row -"skip failed rows" handlers (sync `FunctionNode.execute`, async -`_FunctionPodBase.async_execute.process_one`, side-effect delivery with `on_error="log"`) -re-raise `StoreWriteError` ahead of their generic `except Exception`, so `error_policy` keeps -meaning "skip rows whose *function* failed". The async orchestrator unwraps the -`TaskGroup` `ExceptionGroup` so async callers get the same `StoreWriteError` as sync callers. -`PipelineJob.run()` flushes pending writes best-effort before re-raising any run failure, so -rows that did write are committed and do not look dropped. - -**Tech Stack:** Python 3.11+, PyArrow, asyncio TaskGroup, pytest. - ---- - -## Investigation summary (Step 1, reproduced on main c438d4d1) - -- Sync swallow: `core/nodes/function_node.py` `FunctionNode.execute` per-row - `except Exception` (error_policy "continue") wraps `_process_data_internal`, which does - compute AND both writes (result table via `ResultCache.store`, pipeline table via - `add_pipeline_record`). WARNING log per row, run returns normally. -- Async swallow: `core/function_pod.py` `_FunctionPodBase.async_execute.process_one` catches - every per-row exception, logs at DEBUG (invisible), drops the row. -- Async pipeline-table write (`record_and_forward`) and operator LOG writes already raise, but - async callers get an `ExceptionGroup`. -- Operator-LOG failure aborts the run before `job.run`'s final flush, so pending pipeline rows - of upstream function nodes are never committed while their result rows are: the node reads - back fewer rows than were stored. -- Read failures raise on both orchestrators (not swallowed). Observer status/log writes are - deliberately best-effort and stay so. -- Re-run on a healthy store recovers in every case (missing rows recomputed; orphan results - hit the result cache). - -## Contract (Option A) - -1. A failed write of a node's results or records raises `StoreWriteError` and stops the run at - the first failure, regardless of `error_policy`, on both orchestrators. -2. `str(err)` names the node label and the table path (`"/".join(record_path)`), says rows - stored before the failure are kept and that re-running computes only what is missing. - `err.node_label`, `err.record_path` are attributes; `err.__cause__` is the backend error. -3. Before re-raising, `PipelineJob.run()` flushes pending writes best-effort; a flush failure - during that is logged and does not mask the original error. Nothing already stored is - rewritten; no hash, path or stored-schema change. -4. A failure of `PipelineJob.run()`'s final flush raises `StoreWriteError` (no node label). -5. `error_policy="continue"` still skips rows whose function raised (unchanged). - -## Files - -- Modify: `src/orcapod/errors.py` — add `StoreWriteError(RuntimeError)`. -- Modify: `src/orcapod/core/result_cache.py` — wrap `add_record` + auto-flush in `store`. -- Modify: `src/orcapod/core/nodes/function_node.py` — wrap `add_pipeline_record`'s write; - re-raise `StoreWriteError` in `execute`; tag node label in `async_execute`. -- Modify: `src/orcapod/core/function_pod.py` — `process_one` re-raises `StoreWriteError`. -- Modify: `src/orcapod/core/nodes/operator_node.py` — wrap `_store_output_stream`'s write. -- Modify: `src/orcapod/side_effects.py` — wrap `_write_invocation_row`; re-raise before the - `on_error="log"` handler. -- Modify: `src/orcapod/pipeline/async_orchestrator.py` — unwrap `StoreWriteError` from the - `ExceptionGroup`. -- Modify: `src/orcapod/pipeline/job.py` — best-effort flush on failure; wrap final flush. -- Create: `tests/test_pipeline/test_store_write_failure.py`. -- Modify: `docs/concepts/function-pods.md` (PipelineJob section), `README.md` execution - walkthrough, `DESIGN_ISSUES.md` (one entry appended at the end). - -## Task 1: Failing regression tests (public interface) - -**Files:** Create `tests/test_pipeline/test_store_write_failure.py`. - -Test double `FaultyStore`: wraps an `InMemoryArrowDatabase`; `at()` returns a wrapper sharing -one `Faults` config; fails `add_record`/`add_records` whose full path contains a marker -(`"rdb_v1"` result table, `"pdb_v1"` pipeline table, operator class name), optionally after N -successful matching writes, or fails `flush`. Records the failing relative record path so the -test can assert the message names it. `__getattr__` delegates everything else. - -All tests parametrized over `SyncPipelineOrchestrator` / `AsyncPipelineOrchestrator`. Until -the exception class is approved, assert with `pytest.raises(Exception)` plus -`not isinstance(err, BaseExceptionGroup)`, node label in `str(err)` and failing path in -`str(err)`; tighten to `StoreWriteError` in Task 2. - -- result-table write fails on the first row → run raises, names node + table. -- result-table write fails from the 2nd row → raises; a fresh job over the healthy store - recomputes only the missing rows (function call counter) and returns all rows. -- pipeline-table write fails → raises; fresh healthy re-run makes zero function calls. -- store primed with 2 rows, 3rd (new) row's write fails → raises; the 2 primed rows are still - read back by a fresh job (zero calls for them). -- `error_policy="fail_fast"` and `"continue"` both raise for a store failure. -- operator node with `cache_mode=LOG` whose write fails → raises naming the operator node; - upstream function node's pipeline rows are committed (read through a fresh job's - `nodes[label].get_all_records()` before running it). -- flush failure (`Faults(on_flush=True)`) → raises. -- SQLite trigger: function pod returning `pathlib.Path` on - `ConnectorArrowDatabase(SQLiteConnector(tmp_path / "x.db"))` → raises naming the node. -- Guard (should pass before and after): a function that raises for one row with - `error_policy="continue"` still returns normally and stores the other rows. - -- [ ] Write the tests; run `uv run pytest tests/test_pipeline/test_store_write_failure.py -v`; - confirm the store-failure tests FAIL (run returns normally / ExceptionGroup) and the - guard passes. Commit `test(pipeline): failing regression tests for swallowed store write - failures (ITL-878)`. - -## Task 2: `StoreWriteError` (after approval) - -```python -class StoreWriteError(RuntimeError): - """Raised when a node's results or records could not be written to the store. ...""" - - def __init__(self, record_path: tuple[str, ...], node_label: str | None = None) -> None: - self.record_path = tuple(record_path) - self.node_label = node_label - super().__init__(record_path, node_label) - - def __str__(self) -> str: - where = f"node {self.node_label!r}" if self.node_label else "the pipeline" - cause = self.__cause__ - reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" - return ( - f"Could not write to the store for {where} (table " - f"{'/'.join(self.record_path)!r}){reason}. Rows stored before this failure are " - "kept; re-running computes only the rows that are missing." - ) -``` - -`node_label` is filled in by the node that owns the write when the write site does not know -it (`ResultCache`): `if exc.node_label is None: exc.node_label = self.label`. -Tighten Task 1 assertions to `pytest.raises(StoreWriteError)`. - -## Task 3: Write sites raise `StoreWriteError` - -- `ResultCache.store`: `try: add_record(...); if auto_flush: flush()` / - `except StoreWriteError: raise` / `except Exception as exc: raise - StoreWriteError(self._versioned_record_path) from exc`. -- `FunctionNode.add_pipeline_record`: same around `self._pipeline_database.add_record`, with - `node_label=self.label`, path `self._versioned_pipeline_path`. -- `OperatorNode._store_output_stream`: same around `add_records`, path - `self.node_identity_path`. -- `side_effects._write_invocation_row`: same, path `table_path` (label filled by caller). - -## Task 4: Per-row handlers re-raise - -- `FunctionNode.execute`: before `except Exception`, add - `except StoreWriteError as exc:` → set label, `ctx_obs.on_data_crash(...)`, - `ctx_obs.on_node_end(...)`, `raise`. -- `_FunctionPodBase.async_execute.process_one`: `except StoreWriteError as exc:` - → `obs.on_data_crash(...)`; `raise`. (Only this; no error_policy threading or log-level - change — those are the separately filed async error_policy issue.) -- `FunctionNode.async_execute`: wrap the DB-path `TaskGroup` with `except* StoreWriteError` - setting `node_label` on each leaf whose label is `None`, then re-raise the group. -- `side_effects` delivery: `except StoreWriteError: raise` before the `on_error` handler. - -## Task 5: Orchestrator and job - -- `AsyncPipelineOrchestrator._run_async`: `except* StoreWriteError as eg: raise - _first_leaf(eg)` where `_first_leaf` walks nested groups to the first `StoreWriteError`. -- `PipelineJob.run()`: wrap `effective_orchestrator.run(...)` in `try/except Exception:` - → `_flush_best_effort(pipeline_db, result_db)` (logs flush errors with `logger.exception`) - → `raise`. Final flush: `except Exception as exc: raise - StoreWriteError(pipeline_db.base_path) from exc`. - -- [ ] Run new tests (pass), then `uv run pytest tests/` (full), ruff check/format, - basedpyright exit code. Commit `fix(pipeline): a failed store write fails the run - (ITL-878)`. - -## Task 6: Docs and design log - -- `docs/concepts/function-pods.md` PipelineJob section: subsection "When the store cannot be - written" stating the contract (prose; no executed sample needs infrastructure). -- `README.md` execution walkthrough: note store errors are not skipped by `error_policy`. -- `DESIGN_ISSUES.md`: append one entry (resolved, Fix note, ITL-878). -- [ ] `uv run pytest tests/test_docs/`; commit `docs: state the store-write failure contract - (ITL-878)`. - -## Out of scope (reported, not fixed) - -- Async orchestrator ignores `error_policy`; async per-row function errors logged at DEBUG - (coordinator is filing). This plan touches `process_one` only to re-raise - `StoreWriteError`. -- Connector/Delta `flush` pop the pending batch before writing (coordinator is filing). -- ITL-877 (SQLite logical types). - ---- - -## Follow-up: CodeRabbit round 1 (approved by the coordinator if each finding reproduces) - -**Finding 1 (comment 4534032564): a failing hook or observer callback replaces a -`StoreWriteError`.** A node's own `CachedFunctionPod` never carries hooks, so the reachable -case is a node function that runs another cached pod (`@function_pod(pod_cache_database=..., -post_run_hooks=[...])`) whose store write fails: the ERROR-status hook fires inside -`_invoke_with_hooks`'s `except`, a fail-loud hook raises, its exception replaces the store -error, and the outer node skips the row under `error_policy="continue"`. The same applies to -an observer's `on_data_crash` / `on_node_end` raised on the store-error branch. - -- Test first (public interface): standalone `pod.pod(stream).as_table()` and - `pod.pod.async_execute(...)` with a raising hook raise `StoreWriteError`; a pipeline whose - node function runs such a pod raises `StoreWriteError` on both orchestrators under - `continue`; a pipeline observer whose `on_data_crash` raises does not hide it. -- Fix: `_fire_post_run_hooks(payload, store_error=...)`: when a store error is propagating, - a raising hook is logged at WARNING and recorded with `add_note` on the store error, which - then propagates. Observer callbacks on the store-error branches go through one helper that - does the same. Hook failures without a store error are unchanged. - -**Finding 2 (comment 4534032576): mixed exception groups from the async orchestrator.** -`except* StoreWriteError` re-raises the unmatched rest as an `ExceptionGroup` together with -the store error, and drops further store errors silently. - -- Test first: two sibling nodes failing concurrently (store write + another exception; two - store writes) → the async caller gets one plain `StoreWriteError`. -- Fix: catch `ExceptionGroup`; if it holds any `StoreWriteError`, raise the one whose node - comes first in topological order (ties: group order), the order the sync orchestrator meets - them; log every other `StoreWriteError` at WARNING; add a note per other exception; the - whole group stays reachable as `__context__` (`__cause__` stays the backend error). A - group with no store error propagates unchanged. - -No hash, schema or other behaviour change. - ---- - -## Follow-up: CodeRabbit round 2, out-of-diff findings (review of 0940f995) - -Three findings sat in the review body ("Outside diff range comments"), not in threads. - -**Finding A (job.py final flush, approved if it reproduces): the final-flush `StoreWriteError` -does not warn that pending output may not be stored.** All views of one store share the -pending batches, so `pipeline_db.flush()` commits every table; a batch that fails stops the -flush and the batches after it (a `cache_mode=LOG` operator's output, other nodes' rows) -stay pending, and `result_db.flush()` never runs. - -- Test first: `Faults(on_flush=True)` with the LOG operator; the raised `StoreWriteError` - carries a note saying writes still pending may not have been stored. -- Fix: build the error, `add_note(...)`, `raise err from exc`. Same error class, same - `record_path`, same `__cause__`. - -**Finding B (job.py `_flush_best_effort`, approved if it reproduces): a failed best-effort -flush shows only in the log.** - -- Test first: an operator LOG write fails and the best-effort flush fails too; the raised - `StoreWriteError` is still the operator's, and carries a note that the flush failed and - pending results may not have been stored (upstream rows read back: 0). -- Fix: `_flush_best_effort(run_error, *databases)` keeps logging and adds a note naming the - flush failure to the propagating run error. The raised error is unchanged. - -**Finding C (`ResultCache.store` record preparation): REPRODUCE AND REPORT ONLY.** Establish -on both orchestrators and both error policies which function outputs fail in record -preparation (naive datetime, value not matching the declared type, unsupported type, a -logical type whose `python_to_storage` raises), what the caller sees (row skipped? WARNING -log? observer `on_data_crash`? run returns? anything stored?), and what CodeRabbit's -proposal would change. Ask the coordinator to choose (a) row failure under `error_policy` -reported exactly like a function failure, or (b) `StoreWriteError`. No fix before the reply. - -No hash, schema or stored-data change. - ---- - -## Follow-up: Finding C decided — an unstorable output is a row failure (2026-11-11) - -**Decision (developer, November 11, 2026):** a function output that cannot be turned into a -storable record (naive datetime, `str` for a declared `int`, `int` for a declared dataclass) -is a ROW FAILURE governed by `error_policy`, exactly like a function that raised. It is not a -`StoreWriteError`. No behaviour change. - -Resumed after a machine restart; the previous worker's local commit ff4c25cb (12 pinning -tests) is kept. - -- [x] Pinning tests (`TestUnstorableOutputIsARowFailure`, sync/async x continue/fail_fast x - 3 outputs, each compared with a function raising on the same row) also assert the - per-row log record (level + message) matches the function-failure one, besides the - observer `on_data_crash`, raise-or-return, rows and stored rows. -- [x] Docs (`docs/concepts/function-pods.md`, "When the store cannot be written"): store - failures fail the run; a function output that cannot be stored is a row failure; - async logs row failures at DEBUG and ignores `fail_fast` (ITL-879), unchanged here. -- [x] `DESIGN_ISSUES.md` SW1: the same rule. -- [x] Decline finding (1) with a signed top-level PR comment (issuecomment-6791981712). -- [x] Merged origin/main b197544e (DESIGN_ISSUES conflict: both appended sections kept); - added the `[0.3.0] — unreleased` CHANGELOG section with this PR's entries. -- [x] Merge origin/main (b197544e; CHANGELOG entries under `[0.3.0] — unreleased`), full - suite on 3.11/3.12/3.13, golden unchanged, ruff, format, basedpyright, push, CI, - CodeRabbit round (bodies too). Reconcile with #290 if it merged first. - ---- - -## Follow-up: CodeRabbit round 3 (review of d305c806), approved by the coordinator 2026-11-11 - -- (A) docs: a normal return stores every result that could be turned into a record; point to - the row-failure paragraph. -- (B) docs: a side effect whose delivery completed but whose invocation-log write failed is - delivered again on re-run (track_completion has no record); check or make it idempotent. -- (C) `ResultCache.store`: separate `add_record` from auto-`flush`. A flush failure raises - `StoreWriteError(result_db.base_path, flush=True)`; new keyword-only `flush: bool` - attribute; `add_context` then sets the node label but not the row tag; message says pending - writes could not be committed and the failing write may belong to an earlier row or another - table. `job.py` final flush uses `flush=True`. Test first. -- (D) `StoreWriteError` message: "Rows stored before this failure are kept; re-running the job - does not recompute stored function results." Docs explain LOG operators recompute. -- Keep shape consistent with #290's `StoreReadError`. -- [x] tests, fix (392ee8ed), replies on the 4 threads. -- [ ] CI and CodeRabbit on the final head, worker_done. diff --git a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md deleted file mode 100644 index 54b3aaa6c..000000000 --- a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md +++ /dev/null @@ -1,189 +0,0 @@ -# User class identity (v0.3) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Identify a dataclass or pydantic model by `{kind, name, fields}` (spec -`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`), with a declared-name API, silent -rebinding of identical re-registrations, and a raise on stored/registered shape mismatch. - -**Architecture:** One new module, `orcapod/logical_types/user_class_identity.py`, owns the -naming policy, declarations, type tokens, the identity record and string (single-point -format), the process-wide `(kind, name)` table and the read-side class lookup. The two -factories share a new base (`user_class_logical_type.py`) that builds the identity string -and uses it as logical type name and Arrow extension name; the metadata carries the record. -`LogicalTypeRegistry` learns to rebind an identical identity to a new class object and stops -matching user classes by subclass. The public API (`register_type`, `set_type_naming`) lives -in `orcapod/user_types.py`. `TypeIdentityError` lives in `orcapod/errors.py`; the Delta read -path re-raises it. - -**Tech Stack:** Python 3.11–3.13, pyarrow, polars, pydantic v2, deltalake, IPython (dev, for -the notebook scenarios), pytest via `uv run`. - -**Changes after the plan was written (developer decisions, November 10, 2026):** -`set_type_naming` was dropped (fixed location rule plus `register_type(qualified_name=False)`), -the identity string uses the full 64-hex digest, and stored structs list fields in name -order. The spec is authoritative where this plan differs. - ---- - -## File map - -| File | Change | -|---|---| -| `src/orcapod/errors.py` | add `TypeIdentityError(ValueError)` | -| `src/orcapod/logical_types/user_class_identity.py` | **new**: kinds, naming policy, declarations, `type_token`, `identity_record`, `canonical_json`, `identity_string`, known-class table, `find_class`, `describe_fields`, S3 notice | -| `src/orcapod/logical_types/user_class_logical_type.py` | **new**: `UserClassLogicalType`, `UserClassLogicalTypeFactory` (shared write/read path) | -| `src/orcapod/logical_types/dataclass_logical_type_factory.py` | reduce to subclasses: kind, category, `supports_class`, stored fields | -| `src/orcapod/logical_types/pydantic_logical_type_factory.py` | same | -| `src/orcapod/logical_types/registry.py` | rebinding rule; exact-type match for user classes; `TypeIdentityError` from `_deserialize` | -| `src/orcapod/semantic_types/universal_converter.py` | cycle error names the class and says recursive classes cannot be stored | -| `src/orcapod/databases/delta_lake_databases.py` | re-raise `TypeIdentityError` where read errors are logged | -| `src/orcapod/logical_types/database_hooks.py` | add the column name to a `TypeIdentityError` raised while registering a discovered type | -| `src/orcapod/user_types.py` | **new**: `register_type`, `set_type_naming` | -| `src/orcapod/__init__.py` | re-export the two functions | -| tests | `tests/test_user_types/` (new): unit tests of identity, API, registry, read path; `test_workshop_scenarios.py` (subprocess + IPython `InteractiveShell`) | -| existing tests asserting `module.qualname` names | switch to the identity string | -| `tests/test_hashing/golden/cases.py` + `data/*.json` | add new fixture cases; regenerate (25 listed moves + additions) | -| docs | new `docs/concepts/your-own-data-types.md`; section in `docs/concepts/identity.md`; nav in `mkdocs.yml`; link from `function-pods.md` | -| `CHANGELOG.md`, `DESIGN_ISSUES.md` | entry each | - -## Key definitions (used by every task) - -```python -# user_class_identity.py -KIND_DATACLASS = "dataclass" -KIND_PYDANTIC = "pydantic" -_NAMING = {"main": "name", "module": "qualified"} # set_type_naming -_MAIN_MODULES = frozenset({"__main__", "__mp_main__"}) -_DECLARED: weakref.WeakKeyDictionary[type, str] # register_type -_KNOWN: dict[tuple[str, str], list[weakref.ref[type]]] # (kind, name) -> classes - -def validate_name(name: str) -> None: ... # non-empty, no '@', no whitespace, not 'orcapod.' -def class_kind(cls: type) -> str | None: ... -def resolve_name(cls: type) -> str: ... # declared > main policy > module policy -def declare(cls: type, name: str | None) -> None: ... # raises if already built under another name -def set_naming(main: str | None, module: str | None) -> None: ... -def type_token(annotation, converter) -> str: ... -def identity_record(kind, name, fields: dict[str, str]) -> dict: ... -def canonical_json(obj) -> str: ... # sort_keys, (",", ":") -def identity_string(record) -> str: ... # f"{name}@{sha256(canonical_json)[:16]}" (single point) -def describe_fields(fields: dict[str, str]) -> str: ... # "{x: int, y: float}" -def remember(kind, name, cls) -> None: ... # _KNOWN -def candidates(kind, name) -> list[type]: ... # _KNOWN + declared + __main__ walk + import -``` - -Type tokens: builtins bare (`int`); `NoneType` → `None`; other classes without logical type → -`canonical_type_name`; class with logical type → `lt.logical_type_name` (built-ins -`orcapod.path`, user classes their identity string, P1); `list[...]`/`set[...]`/`dict[...]` -→ `origin[args]` with origin by canonical name minus `builtins.`; unions → sorted members -joined by ` | `; `Literal` → `Literal[...]` with sorted `repr`s. - ---- - -### Task 1: Identity primitives (pure functions) - -**Files:** create `src/orcapod/logical_types/user_class_identity.py`, modify -`src/orcapod/errors.py`; test `tests/test_user_types/test_identity_record.py`. - -- [ ] Step 1: failing tests: `identity_string({"kind":"dataclass","name":"Point","fields":{"x":"int","y":"float"}}) == "Point@9117beaaf7ebe3ab"`; field-order independence; `type_token` for `int`, `int | None`, `Optional[int]`, `None | int`, `list[int]`, `typing.List[int]`, `dict[str, float]`, `Path` (`orcapod.path`), `datetime.datetime`, `Literal["b","a"]`; `validate_name` rejects `""`, `"a@b"`, `"a b"`, `"orcapod.x"`; `resolve_name` for a module class (`tests.…Point` qualified), a class with `__module__="__main__"` (`"Point"`), `"__mp_main__"` (same), after `set_naming(main="qualified")` (`"__main__.Point"`), after `set_naming(module="name")`. -- [ ] Step 2: run, expect ImportError (module missing). -- [ ] Step 3: implement the functions above. -- [ ] Step 4: run, pass. Commit `feat(types): identity record and string for user classes (ITL-881)`. - -### Task 2: Shared user-class logical type; factories use the identity - -**Files:** create `src/orcapod/logical_types/user_class_logical_type.py`; reduce both factory -modules to subclasses; test `tests/test_user_types/test_user_class_logical_type.py`. - -`UserClassLogicalType(identity, record, python_type, storage_type, field_annotations, category)`: -logical name = Arrow extension name = identity string; metadata -`canonical_json({"category": category, "identity": record}).encode()`; conversions as today. -Class attribute `exact_python_type_only = True`. - -`UserClassLogicalTypeFactory` (abstract: `kind`, `category`, `supports_class`, `stored_fields(cls, hints)`): -- `create_for_python_type(cls, converter)`: reject ``; name = `resolve_name(cls)`; - for each stored field: `arrow = converter.register_python_class(annotation)` (wrap a - circular `TypeError` as `TypeError(f"{cls.__qualname__}.{field}: recursive classes cannot be stored ({exc})")`); - token = `type_token(annotation, converter)`; record; identity; `remember(kind, name, cls)`; - S3 notice when the name came from the `__main__` policy (once per name per process). -- `reconstruct_from_arrow(ext_name, storage, metadata, converter)`: no `identity` key → - `TypeIdentityError` (pre-0.3 column); else for each candidate class of (kind, name): get its - logical type through `converter.register_python_class(cls)` + - `converter.get_logical_type(cls)`; return the first whose `logical_type_name == ext_name` - (and whose storage equals the stored storage after `normalize_to_large_types`, else - `TypeIdentityError` naming both struct types). No candidate → `ImportError` naming name, - kind, stored fields and `op.register_type(Point, name='Point')`. Candidates but no match → - `TypeIdentityError` naming class, kind, stored fields, registered fields. - -- [ ] Step 1: failing tests: logical name is the identity string; metadata contains the record; two classes with permuted fields share the identity; dataclass vs pydantic same name+fields differ; same name, `y: int` vs `y: float` differ; nested dataclass token is the inner identity string and changes when the inner fields change or the inner is declared under another name (P1); outer built before inner declared gives the same identity as inner-first (order); self-referential class → `TypeError` naming `Node.next`; `` → `ValueError`; subclass with an extra field gets its own identity (exact-type match). -- [ ] Step 2: run, fail. Step 3: implement. Step 4: update the existing factory/roundtrip tests that assert `module.qualname` (listed in the file map) to read the identity from the registry. Step 5: run `uv run pytest tests/test_logical_types tests/test_user_types -q`, pass. Commit. - -### Task 3: Registry rebinding and exact-type match - -**Files:** `src/orcapod/logical_types/registry.py`; test `tests/test_user_types/test_rebinding.py`. - -- `register_logical_type`: when the logical name is bound to a different instance whose - Arrow extension type has the same name, storage type and serialized metadata, and the - Python type differs → rebind: replace the logical-name and Arrow-name bindings, add the new - Python-type binding, keep the old Python-type binding. If the previous class's - `module.qualname` differs from the new one's → one `UserWarning` naming both. Anything - else keeps raising. -- `get_by_python_type`: skip the `issubclass` fallback for logical types with - `exact_python_type_only`. -- `_deserialize`: raise `TypeIdentityError` (subclass of `ValueError`) for storage/metadata mismatch. - -- [ ] Steps: failing tests (rebind same identity silently, `warnings` none; different module.qualname same identity → one `UserWarning`; one class object under two names → `ValueError`), implement, pass, commit. - -### Task 4: Public API - -**Files:** create `src/orcapod/user_types.py`; modify `src/orcapod/__init__.py`; test -`tests/test_user_types/test_register_type.py`. - -```python -def register_type(cls=None, /, *, name=None): # decorator, decorator factory, or call -def set_type_naming(*, main=None, module=None) -> None: -``` - -- [ ] Steps: failing tests through `op.register_type` / `op.set_type_naming` and a `DictSource` + `function_pod` pipeline: declared name gives the identity `Name@…`; `register_type(Point, name="Geo")` after first use under another name → `ValueError`; invalid names; non-dataclass → `TypeError`; returns the class. Implement, pass, commit. - -### Task 5: Read path and S2 guard - -**Files:** `delta_lake_databases.py`, `database_hooks.py`; test -`tests/test_user_types/test_read_path.py` (public interface: pipeline in a subprocess writes -to `DeltaTableDatabase`, a second subprocess reads with a different class). - -- [ ] Steps: failing tests: reader with matching class (declared in another module) reads values; reader without the class → `ImportError` naming `Point` and `register_type`; reader whose `Point` has different fields reading the stored table directly → `TypeIdentityError` naming column, both field sets; re-run of the pipeline with changed fields computes into new tables (calls == rows), old table still readable by the old class. Implement re-raise, pass, commit. - -### Task 6: Workshop scenarios W1–W12 as tests - -**File:** `tests/test_user_types/test_workshop_scenarios.py` (each scenario in a fresh -subprocess; notebook cells through `IPython.core.interactiveshell.InteractiveShell`, as E02; -children end with `os._exit(0)` after printing JSON, as the E-harness does). - -W1 re-run same fields (calls 0 on second run, no error); W2 changed fields (new tables: -calls 2, rows 2, no error); W3 pydantic re-run; W4 `importlib.reload`; W5 restart + re-run -(two processes, cache hit); W6 restart with changed fields (recompute, rows 2); W7 script as -`__main__` twice; W8 `-m` vs import with a declared name (hit); W11 notebook → module with a -declared name: identical pipeline hashes and table paths; W12 read notebook results from a -script defining the same `Point`. Plus: pydantic vs dataclass same name+fields are distinct -(different pod pipeline hashes). - -- [ ] Steps: write, run (must fail before Tasks 1–5 on `main` for W1–W4, W6, W11, W12; record which), pass after, commit. - -### Task 7: Golden values - -- [ ] Add fixture cases to `cases.py` (new names only; nothing moved): declared-name dataclass, nested dataclass, `Optional` field, container field, permuted-order twin pair. -- [ ] Run `uv run pytest tests/test_hashing/golden -q`: exactly the 25 spec entries fail. Record the list. -- [ ] Regenerate; `git diff --stat` and a script comparing old/new JSON: changed keys == the 25, plus only added keys. Commit `test(hashing): golden values for v0.3 user class identity (ITL-881)`. - -### Task 8: Docs, CHANGELOG, DESIGN_ISSUES - -- [ ] `docs/concepts/your-own-data-types.md` (runnable samples: define, declare, re-run, what changes identity), identity section in `docs/concepts/identity.md`, nav entry, link from `function-pods.md`. Run `uv run pytest tests/test_docs -q`. -- [ ] CHANGELOG entry with the hash-change statement from spec §e. -- [ ] DESIGN_ISSUES entry appended at the end (resolved, with Fix note). - -### Task 9: Verification and PR - -- [ ] ruff check/format, basedpyright exit code, full `uv run pytest tests/` on 3.11/3.12/3.13 (`uv run --python 3.1x`). -- [ ] Merge `origin/main` (after #286/#289 land), re-run E08 on the new main, regenerate golden only for the listed entries. -- [ ] PR body: intended hash changes list; mark ready via sensei:create-pr. diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md deleted file mode 100644 index 89c7a0f42..000000000 --- a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md +++ /dev/null @@ -1,165 +0,0 @@ -# ITL-884 follow-up: store errors keep their type through `flush()` and out of the async TaskGroup - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan -> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -> **Superseded in part (November 11, 2026):** the developer's review of PR #290 removed every -> parameter used only for an error message. `_read_delta_table` no longer takes `record_path`, -> `_require_compatible_append(table_uri, ...)` became `_incompatible_append_column(...)`, -> `_store_error_to_raise` takes only the errors, and `_single_store_error` no longer takes -> `topo_order`. See `2026-11-11-itl-884-no-message-only-parameters.md`. - -**Goal:** Answer the two open CodeRabbit threads on PR #290 (ids 4534230764 and 4534230775). -A `StoreReadError` or an append-refusal `InputValidationError` raised while flushing reaches the -caller as itself, so a store read failure in a per-row result-cache flush fails the run under -`error_policy="continue"`. The async orchestrator raises a lone `StoreReadError` even when an -unrelated node failed in the same `TaskGroup`. - -**Architecture:** `DeltaTableDatabase.flush()` keeps attempting every pending batch and logging -each failure. When at least one batch failed with a store error, it raises the first -`StoreReadError` (else the first `InputValidationError`) itself, with a note for every other -failed batch. Only non-store failures keep the `RuntimeError` wrapper. The async orchestrator -replaces `except* StoreReadError` with a plain `except ExceptionGroup` that selects one -`StoreReadError` (`_single_store_error`), notes and logs the rest, and raises it, in the same -shape as PR #287's `_single_store_error` for `StoreWriteError` so the second PR to merge has a -trivial conflict. - -**Tech Stack:** deltalake 1.0.2, pyarrow, asyncio `TaskGroup`, pytest. - -Approved by the coordinator (2026-11-10) on condition that each finding reproduces. Both do: - -- (1b) Run a job, then run it again over two new rows, with the function corrupting the result - table's log (a new log version that requires an unknown reader feature) during its first call. - The per-row lookup still succeeds, the result-cache flush raises `StoreReadError` wrapped in - `RuntimeError`, the sync per-row handler under `continue` skips the row, the next row is - computed, and the run ends with `RuntimeError` from the job's end-of-run flush. -- (2) CI's 3.11 is 3.11.18, so the 3.11.0-3.11.3 single-leaf wrapping does not occur there. The - concurrent case reproduces deterministically on 3.11.18 and 3.13.18: node `dbl` raises - `StoreReadError` in `_load_cached_entries` and node `tri` fails in the observer's - `on_node_start`, both before their first `await`, so both land in one group and `job.run` - raises `ExceptionGroup([StoreReadError, ExceptionGroup([ValueError])])`. - -No hash, path or stored-schema changes. No public API added. - ---- - -## File map - -- `src/orcapod/databases/delta_lake_databases.py`: `flush()` and a module helper - `_store_error_to_raise(errors)`. -- `src/orcapod/pipeline/async_orchestrator.py`: plain `except ExceptionGroup`, helpers - `_single_store_error(group, topo_order)` and `_leaves(group)` (identical to #287's `_leaves`). -- Tests: `tests/test_databases/test_delta_read_failures.py` (assert the bare error types, add a - multi-batch test), `tests/test_pipeline/test_store_read_failure.py` (flush during per-row - processing, both orchestrators; concurrent unrelated failure, async). -- Docs: `docs/concepts/function-pods.md` admonition, `CHANGELOG.md` ITL-884 entries, - `DESIGN_ISSUES.md` DR1 fix note. - -## Task 1: Failing tests for finding (1) - -- [ ] `tests/test_databases/test_delta_read_failures.py`: the `refusal` helper is replaced by - `pytest.raises(InputValidationError)` at each refusal site; the two unloadable-table flush tests - use `pytest.raises(StoreReadError)`; the absent `__cause__` assertions go. -- [ ] New `test_flush_raises_the_store_error_and_notes_every_other_failed_batch`: two tables - with pending batches (one whose stored `x` is int64 and receives float64, one whose log requires - an unknown reader feature); `flush()` raises the `StoreReadError`, one note names the other - table's key and `InputValidationError`, and both batches stay queued (a second `flush()` raises - again). -- [ ] New `test_flush_keeps_runtime_error_for_other_failures`: control case. Monkeypatch - `deltalake.write_deltalake` to raise `OSError` for a new table; `flush()` raises `RuntimeError` - with the `OSError` as `__cause__` (unchanged behaviour). -- [ ] `tests/test_pipeline/test_store_read_failure.py`: - `test_store_read_failure_in_a_row_flush_fails_the_run`, both orchestrators with - `error_policy="continue"`. Rerun over `a..e` with the function corrupting the result table on - its first call; `job.run` raises `StoreReadError` naming the result table, and the function ran - for exactly one new row. -- [ ] Run `uv run pytest tests/test_databases/test_delta_read_failures.py - tests/test_pipeline/test_store_read_failure.py -v`. Expected: the new and changed tests FAIL with - `RuntimeError`, and the per-row test also shows two calls. - -## Task 2: Fix `flush()` - -```python -def flush(self) -> None: - errors: list[tuple[str, Exception]] = [] - for record_key in list(self._pending_batches.keys()): - try: - self._flush_by_key(record_key) - except Exception as e: - logger.error(f"Error flushing batch for {record_key}: {e}") - errors.append((record_key, e)) - if errors: - store_error = _store_error_to_raise(errors) - if store_error is not None: - raise store_error - failed_keys = [k for k, _ in errors] - raise RuntimeError( - f"Failed to flush {len(errors)} batch(es): {failed_keys}" - ) from errors[0][1] - - -def _store_error_to_raise(errors): - """First StoreReadError, else first InputValidationError, with notes; else None.""" - for kind in (StoreReadError, InputValidationError): - chosen = next((e for _, e in errors if isinstance(e, kind)), None) - if chosen is not None: - break - else: - return None - for key, other in errors: - if other is not chosen: - chosen.add_note( - f"Another batch also failed to flush ({key}): " - f"{type(other).__name__}: {other}" - ) - return chosen -``` - -`StoreReadError` is preferred because it is the one the per-row handlers treat as fatal. The -docstring gets a Raises section. Run Task 1's tests: PASS. - -## Task 3: Failing test for finding (2) - -- [ ] `test_concurrent_unrelated_failure_still_raises_store_read_error` (async only): an - observer (a `NoOpObserver` subclass whose `contextualize` returns itself and whose - `on_node_start` raises `ValueError` for `tri`), two function nodes over one source, the `dbl` - result table's data files deleted. `job.run` raises `StoreReadError` (not a group) naming the - result table, `__cause__` is set, and a note names `ValueError: observer failed for tri`. -- [ ] Run it: FAIL with `ExceptionGroup`. - -## Task 4: Fix the async orchestrator - -```python - except ExceptionGroup as group: - # A failed store read stops the run. Raise it on its own, as the sync - # orchestrator does, rather than inside the TaskGroup's exception group. - # A plain except (not except*) so the raise is never wrapped again. - store_error = _single_store_error(group, topo_order) - if store_error is None: - raise - raise store_error -``` - -`_single_store_error(group, topo_order)`: `StoreReadError.in_group(group)`; none gives `None`; the first -is chosen; every other leaf (`_leaves`) is noted on it ("Another store read failed in the same -run: ..." also logged at WARNING; "Another node failed in the same run: : "). Its -`__cause__` stays the backend error; the group is its `__context__`. Run: PASS. - -## Task 5: Docs, design log, verification, commit, replies - -- [ ] Concept page and CHANGELOG: `flush()` / `add_records(..., flush=True)` raise - `StoreReadError` and `InputValidationError` themselves; when several batches fail, the first - store error is raised with a note per other batch, every failed batch stays queued and each - failure is logged; other flush failures still raise `RuntimeError`. -- [ ] `DESIGN_ISSUES.md` DR1: one sentence on the flush and async-group fix. -- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, basedpyright (exit code), the full - suite on 3.11, 3.12 and 3.13, `tests/test_hashing/golden` unchanged. -- [ ] One commit per finding, push, signed replies on both threads, wait for CI and CodeRabbit. - -## Interaction with PR #287 (report, do not fix here) - -#287's `ResultCache.store` wraps every `add_record`/`flush` failure in `StoreWriteError`, and its -`PipelineJob.run` wraps the end-of-run flush the same way. Once both PRs are in, a -`StoreReadError` from a flush reaches the caller as the `__cause__` of a `StoreWriteError`. It -is still fatal, but `except StoreReadError` misses it. Whichever PR merges second should decide -whether those wrappers let `StoreReadError` through. diff --git a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md deleted file mode 100644 index e2c5a9fe5..000000000 --- a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md +++ /dev/null @@ -1,210 +0,0 @@ -# ITL-884: Delta read failures are never reported as "no rows" — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan -> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -> **Superseded in part (November 11, 2026):** the developer's review of PR #290 removed every -> parameter used only for an error message. `_read_delta_table` no longer takes `record_path`, -> `_require_compatible_append(table_uri, ...)` became `_incompatible_append_column(...)`, -> `_store_error_to_raise` takes only the errors, and `_single_store_error` no longer takes -> `topo_order`. See `2026-11-11-itl-884-no-message-only-parameters.md`. - -**Goal:** A Delta table that exists but cannot be read raises `StoreReadError`, never an -empty result. A flush that would make an existing table unreadable, or silently cast the -stored column type, is refused before anything is written. - -**Architecture:** Every read path in `DeltaTableDatabase` separates "table absent" -(`TableNotFoundError` → `None`, unchanged) from "table present but unreadable" (raise -`StoreReadError` naming the table URI, with the backend exception chained). `_flush_by_key` -creates a table only on `TableNotFoundError`. Before it merges into an existing table, it compares -the incoming batch's column types with the stored Delta schema. Both sides go through Delta's -own type mapping, with extension types stripped to storage, large-normalized and -nullability-blind. A difference raises `InputValidationError` naming the table, column and both -types. A store read failure fails the run whatever `error_policy` is (same shape as PR #287's -`StoreWriteError`). - -**Tech Stack:** deltalake 1.0.2, pyarrow, pytest. - -Approved by the coordinator (2026-11-10): contract as proposed. D1 adds a sibling -`StoreReadError(RuntimeError)`, and the append refusal raises `InputValidationError`. D2 is -option A: re-raise in the two per-row handlers and unwrap in the async orchestrator, with the -same shape as #287. D3 makes the comparison nullability-blind, plus a test that pins -nullability-only behaviour. D4 fixes the `DeltaProtocolError` → overwrite path. - -No hash, path or stored-schema changes. - ---- - -## File map - -- `src/orcapod/errors.py` gets `StoreReadError` (`table`, `record_path`, `__str__` from - `__cause__`, `in_group`). -- `src/orcapod/databases/delta_lake_databases.py` changes the read paths and - `_flush_by_key`, and adds `_require_compatible_append` and the module helper - `_comparable_storage_type`. -- `src/orcapod/core/nodes/function_node.py` (`execute` per-row handler) and - `src/orcapod/core/function_pod.py` (async `process_one`) both get - `except StoreReadError: ...; raise`. -- `src/orcapod/pipeline/async_orchestrator.py` catches the `TaskGroup`'s `ExceptionGroup` - with a plain `except`, selects one `StoreReadError` with `_single_store_error`, and raises - it on its own, noting the other failures on it. -- Tests: `tests/test_databases/test_delta_read_failures.py` and - `tests/test_pipeline/test_store_read_failure.py`. -- Docs: `docs/concepts/function-pods.md` gets an admonition, `CHANGELOG.md` gets Fixed and - Breaking entries, and `DESIGN_ISSUES.md` gets a new entry at the end. - -## Task 1: Failing regression tests (database level) - -`tests/test_databases/test_delta_read_failures.py`, public `DeltaTableDatabase` API only: - -- `unreadable_table` fixture: write 2 rows, flush, delete the table's `*.parquet` files (the - log stays intact). Use a fresh `DeltaTableDatabase` instance for the reads. -- `test_get_all_records_raises` checks `StoreReadError`: `exc.table` ends with the table - directory, `str(exc)` contains the URI, and `__cause__` is set. -- `test_get_records_by_ids_raises`, `test_get_record_by_id_raises`, - `test_get_records_with_column_value_raises`, and - `test_add_records_skip_duplicates_raises` (existing-ids refresh). Each raises - `StoreReadError`. -- `test_absent_table_reads_return_none` covers all four reads on a path never written: - `None`, `table_exists` is False. -- `test_unloadable_log_raises_on_read_and_flush_never_overwrites`: the table's log names an - unsupported reader feature (the load fails). Reads raise `StoreReadError`. `add_records(..., - flush=True)` raises, the log files and parquet files are byte-identical afterwards, and no - new log version exists. -- `test_protocol_error_on_load_is_not_treated_as_absent`: monkeypatch - `deltalake.DeltaTable` to raise `DeltaProtocolError` for this table (the error deltalake - 1.0.2 does not raise naturally from a local table). Flush raises and the stored files are - untouched. The comment explains the monkeypatch. -- Append refusal: - - `test_castable_type_change_is_refused` (implemented as the `int64->float64` case of - `test_plain_type_change`): int64 stored, float64 appended. `flush` raises - `InputValidationError` itself (Task 2 of `2026-11-10-itl-884-coderabbit-flush-and-async-group.md`); assert it - directly. - The message names the column `x`, `int64` and `double`. A re-read gives the original row - only, with type int64. - - `test_narrowing_change_is_refused`: int64 stored, int32 appended. - - `test_extension_storage_change_is_refused`: `ext` stored, `ext>` - appended. Refused, and the table is still readable afterwards (1 row). - - Legitimate cases still pass: identical schema; `string` vs `large_string` (and - `list` vs `large_list`); a new column; a missing column; an extension type - with unchanged storage (2 rows read back). - - `test_nullability_only_difference_is_accepted`: a nested non-null struct field vs a - nullable one is accepted. This pins today's nullability-blind behaviour (ITL-842 audit). - It also records what Delta stores for nullability after the merge, as an assertion on - the read-back schema. - -Run `uv run pytest tests/test_databases/test_delta_read_failures.py -v`. Expected: the read -tests FAIL (`None` / no raise / raw errors), and the refusal tests FAIL (the write is -accepted). - -## Task 2: Failing regression tests (pipeline level, both orchestrators) - -`tests/test_pipeline/test_store_read_failure.py`, parametrized over -`SyncPipelineOrchestrator()` and `AsyncPipelineOrchestrator()`: - -- Run `source -> FunctionPod(double) label="dbl"` on a `DeltaTableDatabase`, then delete - the parquet files of the `_result` table. -- `test_unreadable_result_table_fails_the_run`: a fresh store and job, `job.run(orchestrator=...)` - raises `StoreReadError`, and the function is not called. -- `test_unreadable_result_table_fails_the_run_under_continue`: also delete the pipeline - (tag) table, so that Phase 1 sees no tag rows and the per-row lookup is the read that fails. - With `error_policy="continue"` it still raises `StoreReadError` and makes 0 calls. -- `test_absent_tables_still_compute`: control case. Delete both tables entirely, then the - run computes all 3 rows. - -Run the tests and expect FAIL. - -## Task 3: `StoreReadError` - -Add the class to `src/orcapod/errors.py`: - -```python -class StoreReadError(RuntimeError): - """Raised when a stored table exists but could not be read. ...""" - - def __init__(self, table: str, record_path: tuple[str, ...]) -> None: - self.table = table - self.record_path = tuple(record_path) - super().__init__(table, self.record_path) - - def __str__(self) -> str: - cause = self.__cause__ - reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" - return ( - f"Could not read the stored table at {self.table!r}{reason}. The table exists, " - "so its rows are not treated as missing and the table was not overwritten." - ) - - @classmethod - def in_group(cls, group: BaseExceptionGroup) -> "list[StoreReadError]": ... -``` - -## Task 4: Delta read paths - -- `_get_delta_table`: on `TableNotFoundError` return `None`. On any other exception, drop - the cache entry and `raise StoreReadError(table_uri, full_path) from exc`. -- Add the helper `_read_failed(record_path, exc) -> StoreReadError`, built from - `_get_table_uri(record_path)` and `self._path_prefix + record_path`. -- `get_all_records`: `except Exception as exc: raise self._read_failed(...) from exc` - (replaces the warning-and-skip). -- `get_records_by_ids`, `get_record_by_id`, `get_records_with_column_value`: narrow each - `try` to the read itself, then `raise self._read_failed(...) from exc` (replaces - `return None` / the raw re-raise). -- `_refresh_existing_ids_cache`: wrap the `to_pyarrow_table()` failure the same way. - -## Task 5: Delta flush path - -- In `_flush_by_key`, load only on `TableNotFoundError`, where `delta_table = None`. Any - other exception does `raise StoreReadError(table_uri, abs_path_components) from exc`. The - existing outer handler restores the pending batch. -- Before `merge`: `self._require_compatible_append(table_uri, delta_table, combined_table)`. -- `_comparable_storage_type(t)` is a module-level function. It strips extension types to - storage at every depth, rebuilds struct/list/fixed-size-list/map with nullable children - (comparison only, never stored), and recurses. -- `_require_compatible_append` builds the stored types from - `pa.schema(delta_table.schema().to_arrow())`. The incoming types go through - `deltalake.Schema.from_arrow(pa.schema([pa.field(name, _comparable_storage_type(type))]))` - and `.to_arrow()`. Both sides are passed through `normalize_to_large_types(..., - keep_null=True)` and `_comparable_storage_type`. Only columns present on both sides are - compared. On a mismatch it raises `InputValidationError(f"Cannot append to the stored table - {table_uri!r}: column {name!r} is stored as {stored} but the new rows have {incoming}. ...")`. - -Run the Task 1 and Task 2 tests and expect PASS. - -## Task 6: Run-level handling (same shape as #287) - -- `FunctionNode.execute` per-row handler: put `except StoreReadError as exc:` before - `except Exception`. It calls `ctx_obs.on_data_crash(...)` and `ctx_obs.on_node_end(...)`, - then `raise`. -- `FunctionPod` async `process_one`: put `except StoreReadError as exc:` before - `except Exception`. It calls `obs.on_data_crash(...)` and then `raise`. -- `AsyncPipelineOrchestrator._run_async`: add `except ExceptionGroup as group:` and pick - the error with `_single_store_error(group, topo_order)` (the `StoreReadError` of the - earliest node in topological order, the other failures noted on it); raise it on its own, - or re-raise the group unchanged when it holds no `StoreReadError`. A leaf raised inside - `except*` would be wrapped in a new group whenever another task failed concurrently. - -Run the Task 2 tests and expect PASS. - -## Task 7: Docs, changelog, design issues, full suite - -- `docs/concepts/function-pods.md` gets a `!!! note "When the store cannot be read"` - admonition after "Re-runs compute only what is new". It has no python block. -- `CHANGELOG.md`: a Fixed entry (ITL-884: read failures raise `StoreReadError`; an unloadable - table is never overwritten). Breaking Changes: a castable column type change appended to an - existing Delta table is now refused with `InputValidationError` instead of being silently - cast to the stored type. -- `DESIGN_ISSUES.md`: one self-contained entry at the end with status resolved and a Fix note. -- Then run `uv run ruff format .`, `uv run ruff check .`, - `uv run --with basedpyright basedpyright` (exit code 0) and `uv run pytest tests/`. -- Commit in Conventional Commits style: `test(databases): ...` for the failing tests first, - then `fix(databases): ...`, then `docs: ...`. - -## Resume notes (2026-11-11, after machine restart) - -- Arrived to a clean worktree at d32c84d1, in sync with origin; PR #290 had 0 open threads - but conflicted with main after #291 (ITL-847). -- Merged origin/main (b197544e) as b4387c00: only DESIGN_ISSUES.md conflicted; kept SJ1 (main) - then DR1 (this PR) at the end, additions-only against both sides. CHANGELOG auto-merged - (#291 did not touch it). Pushed. -- Next: full suite on 3.11/3.12/3.13, CI green, one CodeRabbit request, worker_done. diff --git a/superpowers/plans/2026-11-10-itl-888-array-format.md b/superpowers/plans/2026-11-10-itl-888-array-format.md deleted file mode 100644 index 5f5c8ef67..000000000 --- a/superpowers/plans/2026-11-10-itl-888-array-format.md +++ /dev/null @@ -1,275 +0,0 @@ -# Value-based array format (ITL-888) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Store and hash `numpy.ndarray` values in the v0.3 value-based format of -`superpowers/specs/2026-11-10-array-format-v0.3.md`. - -**Architecture:** A NumPy-free module `logical_types/array_format.py` owns the storage struct and -the value digest. `logical_types/numpy_type.py` maps NumPy dtypes to tags and canonicalises -arrays. `NumpyArrayHandler` hashes through the shared digest. Public-interface tests pin -behaviour across operators, orchestrators, databases and processes. - -**Tech Stack:** numpy 2.2.6, pyarrow 23.0.1, polars 1.41.2, deltalake 1.0.2, starfix, pytest. - -**Gating:** Phase 1 was approved by the coordinator on November 10. Phase 2 implements the -developer's decisions on the identity rules, the name `orcapod.ndarray`, the label `ndarray_v1`, -D-PATH and the refused dtypes (November 10 and 11, 2026; listed in the spec's Decisions). - ---- - -## File structure - -| File | Responsibility | -|---|---| -| `src/orcapod/logical_types/array_format.py` (new) | Storage struct, format version, `array_digest_v1`. No NumPy import. Shared with torch (ITL-871). | -| `src/orcapod/logical_types/numpy_type.py` | dtype to tag and back, canonicalisation, refusals, `LogicalNumpyArray` (Phase 2) | -| `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | `NumpyArrayHandler` on the new digest and label (Phase 2) | -| `src/orcapod/core/function_pod.py` (or the converter it calls) | declared `np.ndarray` return check (Phase 2) | -| `src/orcapod/contexts/data/v0.1.json` | changelog line (Phase 2) | -| `tests/test_arrays/` (new package) | public-interface array tests (Phase 1 scaffolding, Phase 2 format tests) | -| `tests/test_logical_types/test_array_format.py` (new) | digest against an independent reference | - ---- - -## Phase 1 (approved) - -### Task 1: storage struct and value digest - -**Files:** -- Create: `src/orcapod/logical_types/array_format.py` -- Test: `tests/test_logical_types/test_array_format.py` - -- [ ] **Step 1: Write the failing test.** The reference is written independently with `struct` - and `hashlib` only, framing per spec C1: - -```python -import hashlib -import struct - -import pyarrow as pa -import pytest - -from orcapod.logical_types.array_format import ( - ARRAY_STORAGE_TYPE, - array_digest_v1, -) - - -def reference_digest(tag: str, shape: tuple[int, ...], data: bytes) -> bytes: - t = tag.encode("utf-8") - p = struct.pack(" - np.ndarray`; `np.ndarray | None` with a `None` row. -- Operators (called `op(stream)`, read through `iter_data`/`as_table`): `Join`, `SemiJoin`, - `MapData`, `SelectDataColumns`, `DropDataColumns`, `PolarsFilter` on a tag, `Batch`, - `GroupBy`, `MergeJoin` with a colliding array column (values preserved as a set, column is - `list[np.ndarray]`). -- Pipeline stores: `PipelineJob` with `double` and `total`, in-memory and Delta, `job.run()` and - `AsyncPipelineOrchestrator().run(job.dag)`; every node's arrays exact. -- Cached re-run: Delta store, two fresh subprocesses; the second calls no pod (call counter - file) and returns identical arrays and identical node content hashes. -- Hash stability: semantic hash of each array, datagram content hash, and a source's - `content_hash()` in three subprocesses with `PYTHONHASHSEED` 0, 1 and random: identical. - -Steps: write each file, run it (`uv run pytest tests/test_arrays -v`), expect PASS on today's -code; fix the test, not the library, if a test is wrong; report any library failure as a -finding. Commit `test(arrays): public-interface array scaffolding (ITL-888)`. - -### Task 3: draft PR - -Push the branch, open a draft PR against `main` (body: `Fixes ITL-888`, `Fixes ITL-883`, -`Refs ITL-874`, the spec link, the trailer line), then -`orca linear attach` and `orca worktree set --worktree current --pr `. - ---- - -## Phase 2 (after decisions; expanded with the decided values before starting) - -Each task is test-first: write the failing tests, show the failure, implement, pass, commit. - -4. **Tag mapping and canonicalisation** (`numpy_type.py`): `dtype_to_tag`, `tag_to_dtype`, - `canonicalise(arr)`. Tests: every row of spec A2, structured with unsorted fields, nested and - subarray fields, refusals (object, `StringDType`, masked, plus the decided ones) naming the - dtype. Structured copies go field by field (NumPy assigns structured dtypes by position). - Byte order handled by one function, so preserved vs canonicalised is a local change. -5. **`LogicalNumpyArray` on the struct**: new extension name and metadata, polars extension, - `python_to_storage`/`storage_to_python` (writeable copy). Tests: stored Arrow type, round - trip per dtype, extension name `numpy.ndarray` no longer registered. -6. **Column name in conversion errors**: refusal raised through `DictSource` and a function pod - names the column and the dtype. -7. **`NumpyArrayHandler`**: `ContentHash(